diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 5e2e911cc884..026b11a2dbcc 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,8 +2,8 @@ See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request -- [ ] I agree to contribute to the project under Apache 2 License. -- [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV +- [x] I agree to contribute to the project under Apache 2 License. +- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable diff --git a/3rdparty/carotene/CMakeLists.txt b/3rdparty/carotene/CMakeLists.txt index aa95956e7f0b..dc780fd64583 100644 --- a/3rdparty/carotene/CMakeLists.txt +++ b/3rdparty/carotene/CMakeLists.txt @@ -52,5 +52,9 @@ if(WITH_NEON) endif() endif() + if(MINGW) + target_compile_definitions(carotene_objs PRIVATE "-D_USE_MATH_DEFINES=1") + endif() + # we add dummy file to fix XCode build add_library(carotene STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} "$" "${CAROTENE_SOURCE_DIR}/dummy.cpp") diff --git a/3rdparty/hal_rvv/hal_rvv.hpp b/3rdparty/hal_rvv/hal_rvv.hpp index 8bb3545aabab..e10a3258e90f 100644 --- a/3rdparty/hal_rvv/hal_rvv.hpp +++ b/3rdparty/hal_rvv/hal_rvv.hpp @@ -5,8 +5,6 @@ #ifndef OPENCV_HAL_RVV_HPP_INCLUDED #define OPENCV_HAL_RVV_HPP_INCLUDED -#include - #include "opencv2/core/hal/interface.h" #ifndef CV_HAL_RVV_071_ENABLED diff --git a/3rdparty/hal_rvv/version/hal_rvv_071.hpp b/3rdparty/hal_rvv/version/hal_rvv_071.hpp index f86f8c363a25..db235d613984 100644 --- a/3rdparty/hal_rvv/version/hal_rvv_071.hpp +++ b/3rdparty/hal_rvv/version/hal_rvv_071.hpp @@ -5,6 +5,8 @@ #ifndef OPENCV_HAL_RVV_071_HPP_INCLUDED #define OPENCV_HAL_RVV_071_HPP_INCLUDED +#include + #include namespace cv { namespace cv_hal_rvv { diff --git a/3rdparty/ippicv/CMakeLists.txt b/3rdparty/ippicv/CMakeLists.txt index 4ef248f3c0d8..05cab2486f78 100644 --- a/3rdparty/ippicv/CMakeLists.txt +++ b/3rdparty/ippicv/CMakeLists.txt @@ -10,7 +10,7 @@ if(HAVE_IPP_ICV) add_definitions(-DICV_BASE) endif() -file(GLOB lib_srcs ${IPP_IW_PATH}/src/*.c) +file(GLOB lib_srcs ${IPP_IW_PATH}/src/*.c ${IPP_IW_PATH}/src/*.cpp) file(GLOB lib_hdrs ${IPP_IW_PATH}/include/*.h ${IPP_IW_PATH}/include/iw/*.h ${IPP_IW_PATH}/include/iw++/*.hpp) # ---------------------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index e73180b73df0..509a6e193a94 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -307,6 +307,10 @@ OCV_OPTION(WITH_GTK "Include GTK support" ON OCV_OPTION(WITH_GTK_2_X "Use GTK version 2" OFF VISIBLE_IF UNIX AND NOT APPLE AND NOT ANDROID VERIFY HAVE_GTK AND NOT HAVE_GTK3) +OCV_OPTION(WITH_FRAMEBUFFER "Include framebuffer support" OFF + VISIBLE_IF UNIX AND NOT APPLE AND NOT ANDROID) +OCV_OPTION(WITH_FRAMEBUFFER_XVFB "Include virtual framebuffer support" OFF + VISIBLE_IF UNIX AND NOT APPLE AND NOT ANDROID) OCV_OPTION(WITH_WAYLAND "Include Wayland support" OFF VISIBLE_IF UNIX AND NOT APPLE AND NOT ANDROID VERIFY HAVE_WAYLAND) @@ -1019,9 +1023,13 @@ foreach(hal ${OpenCV_HAL}) message(STATUS "NDSRVP: Andes GNU Toolchain DSP extension is not open, disabling ndsrvp...") endif() elseif(hal STREQUAL "halrvv") + if(";${CPU_BASELINE_FINAL};" MATCHES ";RVV;") add_subdirectory(3rdparty/hal_rvv/) ocv_hal_register(RVV_HAL_LIBRARIES RVV_HAL_HEADERS RVV_HAL_INCLUDE_DIRS) list(APPEND OpenCV_USED_HAL "HAL RVV (ver ${RVV_HAL_VERSION})") + else() + message(STATUS "HAL RVV: RVV is not available, disabling halrvv...") + endif() else() ocv_debug_message(STATUS "OpenCV HAL: ${hal} ...") ocv_clear_vars(OpenCV_HAL_LIBRARIES OpenCV_HAL_HEADERS OpenCV_HAL_INCLUDE_DIRS) @@ -1444,6 +1452,13 @@ if(WITH_GTK OR HAVE_GTK) endif() endif() +if(WITH_FRAMEBUFFER OR HAVE_FRAMEBUFFER) + status(" Framebuffer UI:" HAVE_FRAMEBUFFER THEN YES ELSE NO) + if(WITH_FRAMEBUFFER_XVFB OR HAVE_FRAMEBUFFER_XVFB) + status(" Virtual framebuffer UI:" HAVE_FRAMEBUFFER_XVFB THEN YES ELSE NO) + endif() +endif() + if(WITH_OPENGL OR HAVE_OPENGL) status(" OpenGL support:" HAVE_OPENGL THEN "YES (${OPENGL_LIBRARIES})" ELSE NO) endif() diff --git a/cmake/OpenCVFindIPP.cmake b/cmake/OpenCVFindIPP.cmake index 2328ef8b435e..eb26bea5a100 100644 --- a/cmake/OpenCVFindIPP.cmake +++ b/cmake/OpenCVFindIPP.cmake @@ -87,9 +87,14 @@ macro(ipp_detect_version) get_filename_component(IPP_INCLUDE_DIRS ${IPP_VERSION_FILE} PATH) set(__msg) + set(IPP_NEW_LAYOUT 0) if(EXISTS ${IPP_ROOT_DIR}/include/ippicv_redefs.h) set(__msg " (ICV version)") set(HAVE_IPP_ICV 1) + elseif(EXISTS ${IPP_ROOT_DIR}/include/ipp/ipp.h) + set(IPP_NEW_LAYOUT 1) + # workaround to enable both layouts + add_definitions(-DIPP_PRESERVE_OLD_LAYOUT) elseif(EXISTS ${IPP_ROOT_DIR}/include/ipp.h) # nothing else() @@ -117,10 +122,18 @@ macro(ipp_detect_version) if(APPLE AND NOT HAVE_IPP_ICV) _ipp_set_library_dir(${IPP_ROOT_DIR}/lib) - elseif(IPP_X64) - _ipp_set_library_dir(${IPP_ROOT_DIR}/lib/intel64) + elseif (IPP_NEW_LAYOUT) + if(IPP_X64) + _ipp_set_library_dir(${IPP_ROOT_DIR}/lib) + else() + _ipp_set_library_dir(${IPP_ROOT_DIR}/lib32) + endif() else() - _ipp_set_library_dir(${IPP_ROOT_DIR}/lib/ia32) + if(IPP_X64) + _ipp_set_library_dir(${IPP_ROOT_DIR}/lib/intel64) + else() + _ipp_set_library_dir(${IPP_ROOT_DIR}/lib/ia32) + endif() endif() macro(_ipp_add_library name) diff --git a/cmake/OpenCVUtils.cmake b/cmake/OpenCVUtils.cmake index 972cc219fad1..5a44548824ec 100644 --- a/cmake/OpenCVUtils.cmake +++ b/cmake/OpenCVUtils.cmake @@ -1992,7 +1992,7 @@ macro(ocv_git_describe var_name path) OUTPUT_STRIP_TRAILING_WHITESPACE ) if(NOT GIT_RESULT EQUAL 0) - execute_process(COMMAND "${GIT_EXECUTABLE}" describe --tags --always --dirty --match "[0-9].[0-9].[0-9]*" --exclude "[^-]*-cvsdk" + execute_process(COMMAND "${GIT_EXECUTABLE}" describe --tags --always --dirty --match "[0-9].[0-9]*.[0-9]*" --exclude "[^-]*-cvsdk" WORKING_DIRECTORY "${path}" OUTPUT_VARIABLE ${var_name} RESULT_VARIABLE GIT_RESULT diff --git a/cmake/checks/framebuffer.cpp b/cmake/checks/framebuffer.cpp new file mode 100644 index 000000000000..77bb60275c94 --- /dev/null +++ b/cmake/checks/framebuffer.cpp @@ -0,0 +1,9 @@ +#include +#include + +int main(void) +{ + XWDFileHeader *xwd_header; + XWDColor *xwd_colors; + return 0; +} diff --git a/doc/pattern_tools/svgfig.py b/doc/pattern_tools/svgfig.py old mode 100755 new mode 100644 index 37eaf77c8320..73a91ce1349c --- a/doc/pattern_tools/svgfig.py +++ b/doc/pattern_tools/svgfig.py @@ -1,20 +1,32 @@ -# svgfig.py copyright (C) 2008 Jim Pivarski -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA -# -# Full licence is in the file COPYING and at http://www.gnu.org/copyleft/gpl.html +# BSD 3-Clause License + +# Copyright (c) 2022, Jim Pivarski +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import re, codecs, os, platform, copy, itertools, math, cmath, random, sys, copy _epsilon = 1e-5 @@ -29,16 +41,24 @@ except NameError: unicode = lambda s: str(s) -try: - xrange # Python 2 -except NameError: - xrange = range # Python 3 +if re.search("windows", platform.system(), re.I): + try: + import _winreg + _default_directory = _winreg.QueryValueEx(_winreg.OpenKey(_winreg.HKEY_CURRENT_USER, + r"Software\Microsoft\Windows\Current Version\Explorer\Shell Folders"), "Desktop")[0] +# tmpdir = _winreg.QueryValueEx(_winreg.OpenKey(_winreg.HKEY_CURRENT_USER, "Environment"), "TEMP")[0] +# if tmpdir[0:13] != "%USERPROFILE%": +# tmpdir = os.path.expanduser("~") + tmpdir[13:] + except: + _default_directory = os.path.expanduser("~") + os.sep + "Desktop" _default_fileName = "tmp.svg" _hacks = {} _hacks["inkscape-text-vertical-shift"] = False +__version__ = "1.0.1" + def rgb(r, g, b, maximum=1.): """Create an SVG color string "#xxyyzz" from r, g, and b. @@ -437,9 +457,12 @@ def __standalone_xml(self, indent, newl): return output - @staticmethod - def interpret_fileName(fileName=None): - return fileName or _default_fileName + def interpret_fileName(self, fileName=None): + if fileName is None: + fileName = _default_fileName + if re.search("windows", platform.system(), re.I) and not os.path.isabs(fileName): + fileName = _default_directory + os.sep + fileName + return fileName def save(self, fileName=None, encoding="utf-8", compresslevel=None): """Save to a file for viewing. Note that svg.save() overwrites the file named _default_fileName. @@ -591,7 +614,7 @@ def template(fileName, svg, replaceme="REPLACEME"): def load(fileName): """Loads an SVG image from a file.""" - return load_stream(open(fileName)) + return load_stream(file(fileName)) def load_stream(stream): """Loads an SVG image from a stream (can be a string or a file object).""" @@ -1848,7 +1871,7 @@ class Poly: piecewise-linear segments joining the (x,y) points "bezier"/"B" d=[(x, y, c1x, c1y, c2x, c2y), ...] Bezier curve with two control points (control points - precede (x,y), as in SVG paths). If (c1x,c1y) and + preceed (x,y), as in SVG paths). If (c1x,c1y) and (c2x,c2y) both equal (x,y), you get a linear interpolation ("lines") "velocity"/"V" d=[(x, y, vx, vy), ...] diff --git a/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.markdown b/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.markdown index 1557118ab6b3..3a59bf6b23ef 100644 --- a/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.markdown +++ b/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.markdown @@ -46,12 +46,12 @@ assert template is not None, "file could not be read, check with os.path.exists( w, h = template.shape[::-1] # All the 6 methods for comparison in a list -methods = ['cv.TM_CCOEFF', 'cv.TM_CCOEFF_NORMED', 'cv.TM_CCORR', - 'cv.TM_CCORR_NORMED', 'cv.TM_SQDIFF', 'cv.TM_SQDIFF_NORMED'] +methods = ['TM_CCOEFF', 'TM_CCOEFF_NORMED', 'TM_CCORR', + 'TM_CCORR_NORMED', 'TM_SQDIFF', 'TM_SQDIFF_NORMED'] for meth in methods: img = img2.copy() - method = eval(meth) + method = getattr(cv, meth) # Apply template Matching res = cv.matchTemplate(img,template,method) diff --git a/doc/tutorials/app/intelperc.markdown b/doc/tutorials/app/intelperc.markdown index 86be5c9ee98a..5175a0ebe5c9 100644 --- a/doc/tutorials/app/intelperc.markdown +++ b/doc/tutorials/app/intelperc.markdown @@ -3,7 +3,7 @@ Using Creative Senz3D and other Intel RealSense SDK compatible depth sensors {#t @tableofcontents -@prev_tutorial{tutorial_orbbec_astra} +@prev_tutorial{tutorial_orbbec_astra_openni} @next_tutorial{tutorial_wayland_ubuntu} ![hardwares](images/realsense.jpg) diff --git a/doc/tutorials/app/kinect_openni.markdown b/doc/tutorials/app/kinect_openni.markdown index d40116a39ca9..ffeee3f64828 100644 --- a/doc/tutorials/app/kinect_openni.markdown +++ b/doc/tutorials/app/kinect_openni.markdown @@ -4,7 +4,7 @@ Using Kinect and other OpenNI compatible depth sensors {#tutorial_kinect_openni} @tableofcontents @prev_tutorial{tutorial_video_write} -@next_tutorial{tutorial_orbbec_astra} +@next_tutorial{tutorial_orbbec_astra_openni} Depth sensors compatible with OpenNI (Kinect, XtionPRO, ...) are supported through VideoCapture diff --git a/doc/tutorials/app/orbbec_astra.markdown b/doc/tutorials/app/orbbec_astra_openni.markdown similarity index 86% rename from doc/tutorials/app/orbbec_astra.markdown rename to doc/tutorials/app/orbbec_astra_openni.markdown index 84921fe121b1..6bbca0f742b3 100644 --- a/doc/tutorials/app/orbbec_astra.markdown +++ b/doc/tutorials/app/orbbec_astra_openni.markdown @@ -1,4 +1,4 @@ -Using Orbbec Astra 3D cameras {#tutorial_orbbec_astra} +Using Orbbec Astra 3D cameras {#tutorial_orbbec_astra_openni} ====================================================== @tableofcontents @@ -9,7 +9,7 @@ Using Orbbec Astra 3D cameras {#tutorial_orbbec_astra} ### Introduction -This tutorial is devoted to the Astra Series of Orbbec 3D cameras (https://orbbec3d.com/index/Product/info.html?cate=38&id=36). +This tutorial is devoted to the Astra Series of Orbbec 3D cameras (https://www.orbbec.com/products/structured-light-camera/astra-series/). That cameras have a depth sensor in addition to a common color sensor. The depth sensors can be read using the open source OpenNI API with @ref cv::VideoCapture class. The video stream is provided through the regular camera interface. @@ -18,7 +18,7 @@ camera interface. In order to use the Astra camera's depth sensor with OpenCV you should do the following steps: --# Download the latest version of Orbbec OpenNI SDK (from here ). +-# Download the latest version of Orbbec OpenNI SDK (from here ). Unzip the archive, choose the build according to your operating system and follow installation steps provided in the Readme file. @@ -70,6 +70,12 @@ In order to use the Astra camera's depth sensor with OpenCV you should do the fo echo "exit" @endcode + @note The last tried version `2.3.0.86_202210111154_4c8f5aa4_beta6` does not work correctly with + modern Linux, even after libusb rebuild as recommended by the instruction. The last know good + configuration is version 2.3.0.63 (tested with Ubuntu 18.04 amd64). It's not provided officialy + with the downloading page, but published by Orbbec technical suport on Orbbec community forum + [here](https://3dclub.orbbec3d.com/t/universal-download-thread-for-astra-series-cameras/622). + -# Now you can configure OpenCV with OpenNI support enabled by setting the `WITH_OPENNI2` flag in CMake. You may also like to enable the `BUILD_EXAMPLES` flag to get a code sample working with your Astra camera. Run the following commands in the directory containing OpenCV source code to enable OpenNI support: @@ -106,7 +112,7 @@ can be read using the OpenNI interface with @ref cv::VideoCapture class. The vid not available through OpenNI API and is only provided via the regular camera interface. So, to get both depth and color frames, two @ref cv::VideoCapture objects should be created: -@snippetlineno samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp Open streams +@snippetlineno samples/cpp/tutorial_code/videoio/openni_orbbec_astra/openni_orbbec_astra.cpp Open streams The first object will use the OpenNI2 API to retrieve depth data. The second one uses the Video4Linux2 interface to access the color sensor. Note that the example above assumes that @@ -119,12 +125,12 @@ For this example, we’ll configure width and height of both streams to VGA reso the maximum resolution available for both sensors, and we’d like both stream parameters to be the same for easier color-to-depth data registration: -@snippetlineno samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp Setup streams +@snippetlineno samples/cpp/tutorial_code/videoio/openni_orbbec_astra/openni_orbbec_astra.cpp Setup streams For setting and retrieving some property of sensor data generators use @ref cv::VideoCapture::set and @ref cv::VideoCapture::get methods respectively, e.g. : -@snippetlineno samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp Get properties +@snippetlineno samples/cpp/tutorial_code/videoio/openni_orbbec_astra/openni_orbbec_astra.cpp Get properties The following properties of cameras available through OpenNI interface are supported for the depth generator: @@ -156,7 +162,7 @@ As there are two video sources that should be read simultaneously, it’s necess threads to avoid blocking. Example implementation that gets frames from each sensor in a new thread and stores them in a list along with their timestamps: -@snippetlineno samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp Read streams +@snippetlineno samples/cpp/tutorial_code/videoio/openni_orbbec_astra/openni_orbbec_astra.cpp Read streams VideoCapture can retrieve the following data: @@ -177,7 +183,7 @@ two video streams may become out of sync even when both streams are set up for t A post-synchronization procedure can be applied to the streams to combine depth and color frames into pairs. The sample code below demonstrates this procedure: -@snippetlineno samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp Pair frames +@snippetlineno samples/cpp/tutorial_code/videoio/openni_orbbec_astra/openni_orbbec_astra.cpp Pair frames In the code snippet above the execution is blocked until there are some frames in both frame lists. When there are new frames, their timestamps are being checked -- if they differ more than a half of @@ -194,5 +200,5 @@ but the depth data makes it easy. ![Depth frame](images/astra_depth.png) The complete implementation can be found in -[orbbec_astra.cpp](https://github.com/opencv/opencv/tree/5.x/samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp) +[openni_orbbec_astra.cpp](https://github.com/opencv/opencv/tree/5.x/samples/cpp/tutorial_code/videoio/openni_orbbec_astra/openni_orbbec_astra.cpp) in `samples/cpp/tutorial_code/videoio` directory. diff --git a/doc/tutorials/app/table_of_content_app.markdown b/doc/tutorials/app/table_of_content_app.markdown index 6671f6b541fb..c4dc34b23d51 100644 --- a/doc/tutorials/app/table_of_content_app.markdown +++ b/doc/tutorials/app/table_of_content_app.markdown @@ -6,6 +6,6 @@ Application utils (highgui, imgcodecs, videoio modules) {#tutorial_table_of_cont - @subpage tutorial_video_input_psnr_ssim - @subpage tutorial_video_write - @subpage tutorial_kinect_openni -- @subpage tutorial_orbbec_astra +- @subpage tutorial_orbbec_astra_openni - @subpage tutorial_intelperc - @subpage tutorial_wayland_ubuntu diff --git a/modules/core/include/opencv2/core/eigen.hpp b/modules/core/include/opencv2/core/eigen.hpp index 231c6805c0ca..d4b1774cf9c2 100644 --- a/modules/core/include/opencv2/core/eigen.hpp +++ b/modules/core/include/opencv2/core/eigen.hpp @@ -287,6 +287,17 @@ void cv2eigen( const Mat& src, } } +template static inline +void cv2eigen( const Mat& src, + Eigen::Matrix<_Tp, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>& dst ) +{ + CV_CheckEQ(src.dims, 2, ""); + dst.resize(src.rows, src.cols); + const Mat _dst(src.rows, src.cols, traits::Type<_Tp>::value, + dst.data(), (size_t)(dst.outerStride()*sizeof(_Tp))); + src.convertTo(_dst, _dst.type()); +} + // Matx case template static inline void cv2eigen( const Matx<_Tp, _rows, _cols>& src, @@ -307,6 +318,17 @@ void cv2eigen( const Matx<_Tp, _rows, _cols>& src, } } +template static inline +void cv2eigen( const Matx<_Tp, _rows, _cols>& src, + Eigen::Matrix<_Tp, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>& dst ) +{ + CV_CheckEQ(src.dims, 2, ""); + dst.resize(_rows, _cols); + const Mat _dst(_rows, _cols, traits::Type<_Tp>::value, + dst.data(), (size_t)(dst.outerStride()*sizeof(_Tp))); + Mat(src).copyTo(_dst); +} + template static inline void cv2eigen( const Mat& src, Eigen::Matrix<_Tp, Eigen::Dynamic, 1>& dst ) diff --git a/modules/core/src/lut.cpp b/modules/core/src/lut.cpp index af560c7051b7..a6b64c9b96ec 100644 --- a/modules/core/src/lut.cpp +++ b/modules/core/src/lut.cpp @@ -55,6 +55,11 @@ static void LUT8u_32s( const uchar* src, const int* lut, int* dst, int len, int LUT8u_( src, lut, dst, len, cn, lutcn ); } +static void LUT8u_16f( const uchar* src, const hfloat* lut, hfloat* dst, int len, int cn, int lutcn ) +{ + LUT8u_( src, lut, dst, len, cn, lutcn ); +} + static void LUT8u_32f( const uchar* src, const float* lut, float* dst, int len, int cn, int lutcn ) { LUT8u_( src, lut, dst, len, cn, lutcn ); @@ -70,7 +75,7 @@ typedef void (*LUTFunc)( const uchar* src, const uchar* lut, uchar* dst, int len static LUTFunc lutTab[CV_DEPTH_MAX] = { (LUTFunc)LUT8u_8u, (LUTFunc)LUT8u_8s, (LUTFunc)LUT8u_16u, (LUTFunc)LUT8u_16s, - (LUTFunc)LUT8u_32s, (LUTFunc)LUT8u_32f, (LUTFunc)LUT8u_64f, 0 + (LUTFunc)LUT8u_32s, (LUTFunc)LUT8u_32f, (LUTFunc)LUT8u_64f, (LUTFunc)LUT8u_16f }; #ifdef HAVE_OPENCL diff --git a/modules/core/src/mean.dispatch.cpp b/modules/core/src/mean.dispatch.cpp index 00eb4a1d9522..05a1c2d146e9 100644 --- a/modules/core/src/mean.dispatch.cpp +++ b/modules/core/src/mean.dispatch.cpp @@ -484,6 +484,10 @@ void meanStdDev(InputArray _src, OutputArray _mean, OutputArray _sdv, InputArray int dcn = (int)mean_mat.total(); CV_Assert( mean_mat.type() == CV_64F && mean_mat.isContinuous() && (mean_mat.cols == 1 || mean_mat.rows == 1) && dcn >= cn ); + + double* dptr = mean_mat.ptr(); + for(k = cn ; k < dcn; k++ ) + dptr[k] = 0; } if (_sdv.needed()) @@ -495,6 +499,11 @@ void meanStdDev(InputArray _src, OutputArray _mean, OutputArray _sdv, InputArray int dcn = (int)stddev_mat.total(); CV_Assert( stddev_mat.type() == CV_64F && stddev_mat.isContinuous() && (stddev_mat.cols == 1 || stddev_mat.rows == 1) && dcn >= cn ); + + double* dptr = stddev_mat.ptr(); + for(k = cn ; k < dcn; k++ ) + dptr[k] = 0; + } if (src.isContinuous() && mask.isContinuous()) @@ -596,23 +605,17 @@ void meanStdDev(InputArray _src, OutputArray _mean, OutputArray _sdv, InputArray if (_mean.needed()) { const double* sptr = s; - int dcn = (int)mean_mat.total(); double* dptr = mean_mat.ptr(); for( k = 0; k < cn; k++ ) dptr[k] = sptr[k]; - for( ; k < dcn; k++ ) - dptr[k] = 0; } if (_sdv.needed()) { const double* sptr = sq; - int dcn = (int)stddev_mat.total(); double* dptr = stddev_mat.ptr(); for( k = 0; k < cn; k++ ) dptr[k] = sptr[k]; - for( ; k < dcn; k++ ) - dptr[k] = 0; } } diff --git a/modules/core/test/test_arithm.cpp b/modules/core/test/test_arithm.cpp index 46ae48370177..8cb85169af7c 100644 --- a/modules/core/test/test_arithm.cpp +++ b/modules/core/test/test_arithm.cpp @@ -3481,7 +3481,7 @@ TEST_P(NonZeroSupportedMatDepth, hasNonZero) INSTANTIATE_TEST_CASE_P( NonZero, NonZeroSupportedMatDepth, - testing::Values(perf::MatDepth(CV_16F), CV_16BF, CV_Bool, CV_64U, CV_64S, CV_32U) + testing::Values(CV_16BF, CV_Bool, CV_64U, CV_64S, CV_32U) ); /////////////////////////////////////////////////////////////////////////////////// @@ -3498,7 +3498,7 @@ TEST_P(LutNotSupportedMatDepth, lut) INSTANTIATE_TEST_CASE_P( Lut, LutNotSupportedMatDepth, - testing::Values(perf::MatDepth(CV_16F), CV_16BF, CV_Bool, CV_64U, CV_64S, CV_32U) + testing::Values(CV_16BF, CV_Bool, CV_64U, CV_64S, CV_32U) ); /////////////////////////////////////////////////////////////////////////////////// @@ -3526,4 +3526,103 @@ INSTANTIATE_TEST_CASE_P( testing::Values(perf::MatDepth(CV_16F), CV_16BF, CV_Bool, CV_64U, CV_64S, CV_32U) ); +CV_ENUM(LutMatType, CV_8U, CV_16U, CV_16F, CV_32S, CV_32F, CV_64F) + +struct Core_LUT: public testing::TestWithParam +{ + template + cv::Mat referenceWithType(cv::Mat input, cv::Mat table) + { + cv::Mat ref(input.size(), CV_MAKE_TYPE(table.type(), ch)); + for (int i = 0; i < input.rows; i++) + { + for (int j = 0; j < input.cols; j++) + { + if(ch == 1) + { + ref.at(i, j) = table.at(input.at(i, j)); + } + else + { + Vec val; + for (int k = 0; k < ch; k++) + { + val[k] = table.at(input.at>(i, j)[k]); + } + ref.at>(i, j) = val; + } + } + } + return ref; + } + + template + cv::Mat reference(cv::Mat input, cv::Mat table) + { + if (table.type() == CV_8U) + { + return referenceWithType(input, table); + } + else if (table.type() == CV_16U) + { + return referenceWithType(input, table); + } + else if (table.type() == CV_16F) + { + return referenceWithType(input, table); + } + else if (table.type() == CV_32S) + { + return referenceWithType(input, table); + } + else if (table.type() == CV_32F) + { + return referenceWithType(input, table); + } + else if (table.type() == CV_64F) + { + return referenceWithType(input, table); + } + + return cv::Mat(); + } +}; + +TEST_P(Core_LUT, accuracy) +{ + int type = GetParam(); + cv::Mat input(117, 113, CV_8UC1); + randu(input, 0, 256); + + cv::Mat table(1, 256, CV_MAKE_TYPE(type, 1)); + randu(table, 0, 127); + + cv::Mat output; + cv::LUT(input, table, output); + + cv::Mat gt = reference(input, table); + + ASSERT_EQ(0, cv::norm(output, gt, cv::NORM_INF)); +} + +TEST_P(Core_LUT, accuracy_multi) +{ + int type = (int)GetParam(); + cv::Mat input(117, 113, CV_8UC3); + randu(input, 0, 256); + + cv::Mat table(1, 256, CV_MAKE_TYPE(type, 1)); + randu(table, 0, 127); + + cv::Mat output; + cv::LUT(input, table, output); + + cv::Mat gt = reference<3>(input, table); + + ASSERT_EQ(0, cv::norm(output, gt, cv::NORM_INF)); +} + + +INSTANTIATE_TEST_CASE_P(/**/, Core_LUT, LutMatType::all()); + }} // namespace diff --git a/modules/core/test/test_mat.cpp b/modules/core/test/test_mat.cpp index 4837d6738c77..8877481f6343 100644 --- a/modules/core/test/test_mat.cpp +++ b/modules/core/test/test_mat.cpp @@ -2263,6 +2263,27 @@ TEST(Core_Eigen, eigen2cv_check_Mat_type) EXPECT_ANY_THROW(eigen2cv(eigen_A, d_mat)); //EXPECT_EQ(CV_64FC1, d_mat.type()); } + +TEST(Core_Eigen, cv2eigen_check_RowMajor) +{ + Mat A(3, 2, CV_32FC1, Scalar::all(0)); + A.at(0,0) = 1.0; + A.at(0,1) = 2.0; + A.at(1,0) = 3.0; + A.at(1,1) = 4.0; + A.at(2,0) = 5.0; + A.at(2,1) = 6.0; + + Eigen::Matrix eigen_A; + EXPECT_NO_THROW(cv2eigen(A, eigen_A)); + + ASSERT_EQ(1.0, eigen_A(0, 0)); + ASSERT_EQ(2.0, eigen_A(0, 1)); + ASSERT_EQ(3.0, eigen_A(1, 0)); + ASSERT_EQ(4.0, eigen_A(1, 1)); + ASSERT_EQ(5.0, eigen_A(2, 0)); + ASSERT_EQ(6.0, eigen_A(2, 1)); +} #endif // HAVE_EIGEN #ifdef OPENCV_EIGEN_TENSOR_SUPPORT diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index c584499ee254..525f9557452f 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -1193,6 +1193,16 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS DepthToSpaceLayer : public Layer { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS SpaceToDepthLayer : public Layer { + public: + static Ptr create(const LayerParams ¶ms); + }; + //! @} //! @} CV__DNN_INLINE_NS_END diff --git a/modules/dnn/src/cuda4dnn/csl/tensor.hpp b/modules/dnn/src/cuda4dnn/csl/tensor.hpp index 1745c60906fb..91f435058899 100644 --- a/modules/dnn/src/cuda4dnn/csl/tensor.hpp +++ b/modules/dnn/src/cuda4dnn/csl/tensor.hpp @@ -271,7 +271,6 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { typename std::enable_if::value, void> ::type reshape_(ForwardItr start, ForwardItr end) { CV_Assert(start != end); - CV_Assert(std::distance(start, end) <= rank()); using ItrValueType = typename std::iterator_traits::value_type; @@ -290,6 +289,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { auto total = std::accumulate(start, end, 1, std::multiplies()); if (total < 0) { /* there is an unknown size */ + CV_CheckEQ(size() % std::abs(total), static_cast(0), "cannot be reshaped"); // must be divisible if (std::abs(total) <= size()) { unknown_size = size() / std::abs(total); total = size(); @@ -304,11 +304,9 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { CV_Error(Error::StsBadArg, "new axes do not preserve the tensor element count"); } - /* we assume the size of the unspecified axes to be one */ - std::fill(std::begin(shape), std::end(shape), 1); - std::copy_backward(start, end, std::end(shape)); - - /* replace the unknown axis with the correct value */ + /* copy shape from given iterator and reshape -1 with deduced value */ + shape.resize(std::distance(start, end)); + std::copy(start, end, shape.begin()); std::replace(std::begin(shape), std::end(shape), size_type(-1), unknown_size); } @@ -622,6 +620,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { auto total = std::accumulate(start, end, 1, std::multiplies()); if (total < 0) { /* there is an unknown size */ + CV_CheckEQ(size() % std::abs(total), static_cast(0), "cannot be reshaped"); // must be divisible if (std::abs(total) <= size()) { unknown_size = size() / std::abs(total); total = size(); @@ -636,11 +635,9 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { CV_Error(Error::StsBadArg, "new axes do not preserve the tensor element count"); } - /* we assume the size of the unspecified axes to be one */ - std::fill(std::begin(shape), std::end(shape), 1); - std::copy_backward(start, end, std::end(shape)); - - /* replace the unknown axis with the correct value */ + /* copy shape from given iterator and reshape -1 with deduced value */ + shape.resize(std::distance(start, end)); + std::copy(start, end, shape.begin()); std::replace(std::begin(shape), std::end(shape), size_type(-1), unknown_size); } @@ -997,7 +994,6 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { typename std::enable_if::value, void> ::type reshape_(ForwardItr start, ForwardItr end) { CV_Assert(start != end); - CV_Assert(std::distance(start, end) <= rank()); using ItrValueType = typename std::iterator_traits::value_type; @@ -1016,6 +1012,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { auto total = std::accumulate(start, end, 1, std::multiplies()); if (total < 0) { /* there is an unknown size */ + CV_CheckEQ(size() % std::abs(total), static_cast(0), "cannot be reshaped"); // must be divisible if (std::abs(total) <= size()) { unknown_size = size() / std::abs(total); total = size(); @@ -1030,11 +1027,9 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { CV_Error(Error::StsBadArg, "new axes do not preserve the tensor element count"); } - /* we assume the size of the unspecified axes to be one */ - std::fill(std::begin(shape), std::end(shape), 1); - std::copy_backward(start, end, std::end(shape)); - - /* replace the unknown axis with the correct value */ + /* copy shape from given iterator and reshape -1 with deduced value */ + shape.resize(std::distance(start, end)); + std::copy(start, end, shape.begin()); std::replace(std::begin(shape), std::end(shape), size_type(-1), unknown_size); } diff --git a/modules/dnn/src/cuda4dnn/primitives/depth_space_ops.hpp b/modules/dnn/src/cuda4dnn/primitives/depth_space_ops.hpp new file mode 100644 index 000000000000..7846881e703d --- /dev/null +++ b/modules/dnn/src/cuda4dnn/primitives/depth_space_ops.hpp @@ -0,0 +1,76 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_DEPTH_SPACE_OPS_HPP +#define OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_DEPTH_SPACE_OPS_HPP + +#include "../../op_cuda.hpp" + +#include "../csl/stream.hpp" +#include "../csl/tensor.hpp" +#include "../csl/tensor_ops.hpp" +#include "../csl/memory.hpp" +#include "../kernels/permute.hpp" + +#include + +namespace cv { namespace dnn { namespace cuda4dnn { + + template + class DepthSpaceOps final : public CUDABackendNode { + public: + using wrapper_type = GetCUDABackendWrapperType; + + DepthSpaceOps(csl::Stream stream_, const std::vector &internal_shape_, + const std::vector &permutation_) + : stream(std::move(stream_)), internal_shape(internal_shape_), + permutation(permutation_) + { + transposed_internal_shape = std::vector(internal_shape.size()); + for (size_t i = 0; i < permutation.size(); i++) { + transposed_internal_shape[i] = internal_shape[permutation[i]]; + } + + size_t num_elements = std::accumulate(internal_shape.begin(), internal_shape.end(), 1, std::multiplies()); + csl::WorkspaceBuilder builder; + builder.require(num_elements); + scratch_mem_in_bytes = builder.required_workspace_size(); + } + + void forward(const std::vector> &inputs, + const std::vector> &outputs, + csl::Workspace &workspace) override { + CV_CheckEQ(inputs.size(), size_t(1), "DepthSpaceOps: only one input is accepted"); + CV_CheckEQ(outputs.size(), size_t(1), "DepthSpaceOps: only one output is accepted"); + + auto input_wrapper = inputs.front().dynamicCast(); + auto input = input_wrapper->getView(); + CV_CheckEQ(input.rank(), size_t(4), "DepthSpaceOps: input needs to be 4-dimensional [N, C, H, W]"); + auto output_wrapper = outputs.front().dynamicCast(); + auto output = output_wrapper->getSpan(); + auto ws_allocator = csl::WorkspaceAllocator(workspace); + auto transposed_internal = ws_allocator.get_tensor_span(transposed_internal_shape.begin(), transposed_internal_shape.end()); + + // Call reshape on input so that it has the correct shape for permutation + input.reshape(internal_shape.begin(), internal_shape.end()); + kernels::permute(stream, transposed_internal, input, permutation); + // Only copying is needed as output already has the expected shape + auto t = csl::TensorView(transposed_internal); + csl::memcpy(output.get(), t.get(), output.size(), stream); + } + + std::size_t get_workspace_memory_in_bytes() const noexcept override { return scratch_mem_in_bytes; } + + private: + csl::Stream stream; + std::vector internal_shape; + std::vector permutation; + std::vector transposed_internal_shape; + + std::size_t scratch_mem_in_bytes; + }; + +}}} // namespace cv::dnn::cuda4dnn + +#endif // OPENCV_DNN_SRC_CUDA4DNN_PRIMITIVES_DEPTH_SPACE_OPS_HPP diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index d0ce9c0057e3..b2946fc9ee04 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -165,6 +165,10 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(Attention, AttentionLayer); CV_DNN_REGISTER_LAYER_CLASS(GroupNormalization, GroupNormLayer); CV_DNN_REGISTER_LAYER_CLASS(Cast, CastLayer); + CV_DNN_REGISTER_LAYER_CLASS(DepthToSpace, DepthToSpaceLayer) + CV_DNN_REGISTER_LAYER_CLASS(SpaceToDepth, SpaceToDepthLayer) + CV_DNN_REGISTER_LAYER_CLASS(DepthToSpaceInt8, DepthToSpaceLayer) + CV_DNN_REGISTER_LAYER_CLASS(SpaceToDepthInt8, SpaceToDepthLayer) CV_DNN_REGISTER_LAYER_CLASS(Crop, CropLayer); CV_DNN_REGISTER_LAYER_CLASS(Eltwise, EltwiseLayer); diff --git a/modules/dnn/src/layers/depth_space_ops_layer.cpp b/modules/dnn/src/layers/depth_space_ops_layer.cpp new file mode 100644 index 000000000000..932bb6517cd9 --- /dev/null +++ b/modules/dnn/src/layers/depth_space_ops_layer.cpp @@ -0,0 +1,482 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include + +// OpenCL backend +#ifdef HAVE_OPENCL +#include "opencl_kernels_dnn.hpp" +#endif + +// OpenVINO backend +#ifdef HAVE_DNN_NGRAPH +#include "../op_inf_engine.hpp" +#include "../ie_ngraph.hpp" +#endif + +// CUDA backend +#ifdef HAVE_CUDA +#include "../op_cuda.hpp" +#include "../cuda4dnn/primitives/depth_space_ops.hpp" +#endif + +// CANN backend +#ifdef HAVE_CANN +#include "../op_cann.hpp" +#endif + +// TIM-VX backend +#ifdef HAVE_TIMVX +#include "../op_timvx.hpp" +#endif + +namespace cv { namespace dnn { + +struct DepthSpaceOps { + MatShape internal_shape; + MatShape transposed_internal_shape; + std::vector permutation; + +#ifdef HAVE_OPENCL + UMat umat_permutation; + UMat umat_internal_strides; + UMat umat_transposed_internal_strides; +#endif + + void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) { + transposed_internal_shape = MatShape(internal_shape.size()); + for (size_t i = 0; i < permutation.size(); i++) { + transposed_internal_shape[i] = internal_shape[permutation[i]]; + } + +#ifdef HAVE_OPENCL + umat_permutation.release(); + umat_internal_strides.release(); + umat_transposed_internal_strides.release(); +#endif + } + + void cpuCompute(const Mat &input, Mat &output) { + const auto output_shape = shape(output); + Mat tmp; + cv::transposeND(input.reshape(1, internal_shape), permutation, tmp); + tmp.reshape(1, output_shape).copyTo(output); + } + +#ifdef HAVE_OPENCL + bool oclCompute(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) { + std::vector inputs, outputs; + + inputs_arr.getUMatVector(inputs); + outputs_arr.getUMatVector(outputs); + + if (umat_permutation.empty() || umat_internal_strides.empty() || umat_transposed_internal_strides.empty()) { + Mat mat_permutation(1, permutation.size(), CV_32S, permutation.data()); + mat_permutation.copyTo(umat_permutation); + + std::vector internal_strides(permutation.size(), 1), transposed_internal_stides(permutation.size(), 1); + for (int i = static_cast(permutation.size()) - 2; i >= 0; i--) { + internal_strides[i] = internal_strides[i + 1] * internal_shape[i + 1]; + transposed_internal_stides[i] = transposed_internal_stides[i + 1] * transposed_internal_shape[i + 1]; + } + Mat mat_internal_strides(1, internal_strides.size(), CV_32S, internal_strides.data()); + mat_internal_strides.copyTo(umat_internal_strides); + Mat mat_transposed_internal_strides(1, transposed_internal_stides.size(), CV_32S, transposed_internal_stides.data()); + mat_transposed_internal_strides.copyTo(umat_transposed_internal_strides); + } + + const auto output_shape = shape(outputs.front()); + UMat tmp = inputs.front().reshape(1, static_cast(internal_shape.size()), internal_shape.data()); + + bool use_half = (inputs_arr.depth() == CV_16F); + std::string permute_options = cv::format("-DDtype=%s", use_half ? "half" : "float"); + ocl::Kernel permute_kernel("permute", ocl::dnn::permute_oclsrc, permute_options); + if (permute_kernel.empty()) { + return false; + } + UMat transposed_tmp(static_cast(transposed_internal_shape.size()), transposed_internal_shape.data(), inputs_arr.depth()); + size_t num_element = static_cast(std::accumulate(internal_shape.begin(), internal_shape.end(), 1, std::multiplies())); + permute_kernel.set(0, static_cast(num_element)); + permute_kernel.set(1, ocl::KernelArg::PtrReadOnly(tmp)); + permute_kernel.set(2, ocl::KernelArg::PtrReadOnly(umat_permutation)); + permute_kernel.set(3, ocl::KernelArg::PtrReadOnly(umat_internal_strides)); + permute_kernel.set(4, ocl::KernelArg::PtrReadOnly(umat_transposed_internal_strides)); + permute_kernel.set(5, static_cast(permutation.size())); + permute_kernel.set(6, ocl::KernelArg::PtrWriteOnly(transposed_tmp)); + if (!permute_kernel.run(1, &num_element, NULL, false)) { + return false; + } + + transposed_tmp.reshape(1, static_cast(output_shape.size()), output_shape.data()).copyTo(outputs.front()); + return true; + } +#endif // HAVE_OPENCL +}; + +class DepthToSpaceLayerImpl CV_FINAL : public DepthToSpaceLayer, public DepthSpaceOps { +public: + DepthToSpaceLayerImpl(const LayerParams ¶ms) { + setParamsFrom(params); + + CV_CheckTrue(params.has("blocksize"), "DepthSpaceLayer: blocksize is required"); + blocksize = params.get("blocksize"); + + auto mode = params.get("mode", "DCR"); + if (mode == "CRD") { + is_crd = true; + permutation = {0, 1, 4, 2, 5, 3}; + } else if (mode == "DCR") { + is_crd = false; + permutation = {0, 3, 4, 1, 5, 2}; + } else { + CV_Error(Error::StsBadArg, cv::format("DepthToSpace: unsupported mode %s\n", mode.c_str())); + } + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE { + return backendId == DNN_BACKEND_OPENCV || + backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH || + backendId == DNN_BACKEND_CUDA || + backendId == DNN_BACKEND_CANN || + (backendId == DNN_BACKEND_TIMVX && is_crd); + } + + virtual bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE { + CV_CheckEQ(inputs.size(), static_cast(1), "DepthSpaceLayer: accepts only one input"); + const auto &input = inputs.front(); + CV_CheckEQ(input.size(), static_cast(4), "DepthSpaceLayer: input needs to be 4-dimensional [N, C, H, W]"); + int batch = input[0], input_depth = input[1], input_height = input[2], input_width = input[3]; + int output_depth = -1, output_height = -1, output_width = -1; + + CV_CheckEQ(input_depth % (blocksize * blocksize), 0, + "DepthSpaceLayer: requires input depth to be a multiple of (blocksize * blocksize)"); + output_depth = input_depth / blocksize / blocksize; + output_height = input_height * blocksize; + output_width = input_width * blocksize; + + outputs.assign(1, MatShape{batch, output_depth, output_height, output_width}); + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE { + std::vector inputs; + inputs_arr.getMatVector(inputs); + + auto input_shape = shape(inputs.front()); + int batch = input_shape[0], input_depth = input_shape[1], input_height = input_shape[2], input_width = input_shape[3]; + if (is_crd) { + internal_shape = MatShape{batch, input_depth / (blocksize * blocksize), blocksize, blocksize, input_height, input_width}; + } else { + internal_shape = MatShape{batch, blocksize, blocksize, input_depth / (blocksize * blocksize), input_height, input_width}; + } + + DepthSpaceOps::finalize(inputs_arr, outputs_arr); + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + // TODO: support 8-bit int in permute kernel + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget) && inputs_arr.depth() != CV_8S, + DepthSpaceOps::oclCompute(inputs_arr, outputs_arr, internals_arr)) + + if (inputs_arr.depth() == CV_16F) { + forward_fallback(inputs_arr, outputs_arr, internals_arr); + return; + } + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + DepthSpaceOps::cpuCompute(inputs.front(), outputs.front()); + } + +#ifdef HAVE_DNN_NGRAPH + virtual Ptr initNgraph(const std::vector> &inputs, + const std::vector> &nodes) CV_OVERRIDE { + using namespace ov::op; + auto input_node = nodes[0].dynamicCast()->node; + std::shared_ptr output_node; + if (is_crd) { + output_node = std::make_shared(input_node, v0::DepthToSpace::DepthToSpaceMode::DEPTH_FIRST, static_cast(blocksize)); + } else { + output_node = std::make_shared(input_node, v0::DepthToSpace::DepthToSpaceMode::BLOCKS_FIRST, static_cast(blocksize)); + } + return Ptr(new InfEngineNgraphNode(output_node)); + } +#endif // HAVE_DNN_NGRAPH + +#ifdef HAVE_CUDA + Ptr initCUDA(void *context_, + const std::vector>& inputs, + const std::vector>& outputs) override { + using namespace cv::dnn::cuda4dnn; + auto context = reinterpret_cast(context_); + std::vector perm(permutation.begin(), permutation.end()); + return make_cuda_node(preferableTarget, std::move(context->stream), internal_shape, perm); + } +#endif // HAVE_CUDA + +#ifdef HAVE_CANN + virtual Ptr initCann(const std::vector> &inputs, + const std::vector> &outputs, + const std::vector> &nodes) CV_OVERRIDE { + CV_CheckEQ(inputs.size(), static_cast(1), "DepthToSpace/CANN: only accepts one input wrapper"); + CV_CheckEQ(nodes.size(), static_cast(1), "DepthToSpace/CANN: only accepts one input node"); + + auto input_tensor_wrapper = inputs.front().dynamicCast(); + auto input_tensor_desc = input_tensor_wrapper->getTensorDesc(); + auto input_node = nodes.front().dynamicCast()->getOp(); + + auto node = std::make_shared(name); + + node->set_attr_block_size(blocksize); + if (is_crd) { + node->set_attr_mode("CRD"); + } else { + node->set_attr_mode("DCR"); + } + node->set_attr_data_format("NCHW"); + + node->set_input_x_by_name(*input_node, input_tensor_wrapper->name.c_str()); + node->update_input_desc_x(*input_tensor_desc); + + auto output_tensor_desc = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + node->update_output_desc_y(*output_tensor_desc); + + return Ptr(new CannBackendNode(node)); + } +#endif + +#ifdef HAVE_TIMVX + virtual Ptr initTimVX(void* timvx_info_, + const std::vector> &inputs, + const std::vector> &outputs, + bool isLast) CV_OVERRIDE { + auto info = reinterpret_cast(timvx_info_); + CV_Assert(info); + auto timvx_graph = info->getGraph(); + CV_Assert(timvx_graph); + auto graph = timvx_graph->graph; + + auto input_wrapper = inputs.front().dynamicCast(); + int input_wrapper_index = -1; + if (input_wrapper->isTensor()) { + input_wrapper_index = timvx_graph->getTensorIndex(input_wrapper->getTensor()); + if (input_wrapper_index == -1) { + auto tmp = input_wrapper->getMat(); + input_wrapper = std::make_shared(tmp); + } + } + if (!input_wrapper->isTensor() || input_wrapper_index == 1) { + auto input_node_quant = Ptr(new tim::vx::Quantization(tim::vx::QuantType::ASYMMETRIC, 1.0f, 0)); + input_wrapper->createTensor(graph, tim::vx::TensorAttribute::INPUT, input_node_quant); + input_wrapper_index = timvx_graph->addWrapper(input_wrapper); + } + + auto output_wrapper = outputs.front().dynamicCast(); + auto output_node_quant = input_wrapper->getTensorQuantization(); + if (isLast) { + auto shape_type = getShapeTypeFromMat(output_wrapper->getMat()); + output_wrapper->setTensorShape(shape_type); + output_wrapper->createTensor(graph, tim::vx::TensorAttribute::OUTPUT, output_node_quant); + } else { + output_wrapper->createTensor(graph, tim::vx::TensorAttribute::TRANSIENT, output_node_quant); + } + int output_wrapper_index = timvx_graph->addWrapper(output_wrapper); + + std::shared_ptr timvx_node; + timvx_node = graph->CreateOperation(blocksize); + std::vector input_wrapper_indices{input_wrapper_index}, output_wrapper_indices{output_wrapper_index}; + return Ptr(new TimVXBackendNode(timvx_graph, timvx_node, input_wrapper_indices, output_wrapper_indices)); + } +#endif + +private: + int blocksize; + + bool is_crd; +}; + +Ptr DepthToSpaceLayer::create(const LayerParams ¶ms) { + return makePtr(params); +} + +class SpaceToDepthLayerImpl CV_FINAL : public SpaceToDepthLayer, public DepthSpaceOps { +public: + SpaceToDepthLayerImpl(const LayerParams ¶ms) { + setParamsFrom(params); + + CV_CheckTrue(params.has("blocksize"), "SpaceToDepthLayer: blocksize is required"); + blocksize = params.get("blocksize"); + + permutation = {0, 3, 5, 1, 2, 4}; + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE { + return backendId == DNN_BACKEND_OPENCV || + backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH || + backendId == DNN_BACKEND_CUDA || + backendId == DNN_BACKEND_CANN || + (backendId == DNN_BACKEND_TIMVX); + } + + virtual bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE { + CV_CheckEQ(inputs.size(), static_cast(1), "SpaceToDepthLayer: accepts only one input"); + const auto &input = inputs.front(); + CV_CheckEQ(input.size(), static_cast(4), "SpaceToDepthLayer: input needs to be 4-dimensional [N, C, H, W]"); + int batch = input[0], input_depth = input[1], input_height = input[2], input_width = input[3]; + int output_depth = -1, output_height = -1, output_width = -1; + + CV_CheckEQ(input_height % blocksize, 0, "SpaceToDepthLayer: requires input height to be a multiple of blocksize"); + CV_CheckEQ(input_width % blocksize, 0, "SpaceToDepthLayer: requires input width to be a multiple of blocksize"); + output_depth = input_depth * blocksize * blocksize; + output_height = input_height / blocksize; + output_width = input_width / blocksize; + + outputs.assign(1, MatShape{batch, output_depth, output_height, output_width}); + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE { + std::vector inputs; + inputs_arr.getMatVector(inputs); + + auto input_shape = shape(inputs.front()); + int batch = input_shape[0], input_depth = input_shape[1], input_height = input_shape[2], input_width = input_shape[3]; + internal_shape = MatShape{batch, input_depth, input_height / blocksize, blocksize, input_width / blocksize, blocksize}; + + DepthSpaceOps::finalize(inputs_arr, outputs_arr); + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + // TODO: support 8-bit int in permute kernel + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget) && inputs_arr.depth() != CV_8S, + DepthSpaceOps::oclCompute(inputs_arr, outputs_arr, internals_arr)) + + if (inputs_arr.depth() == CV_16F) { + forward_fallback(inputs_arr, outputs_arr, internals_arr); + return; + } + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + DepthSpaceOps::cpuCompute(inputs.front(), outputs.front()); + } + +#ifdef HAVE_DNN_NGRAPH + virtual Ptr initNgraph(const std::vector> &inputs, + const std::vector> &nodes) CV_OVERRIDE { + using namespace ov::op; + auto input_node = nodes[0].dynamicCast()->node; + std::shared_ptr output_node; + output_node = std::make_shared(input_node, v0::SpaceToDepth::SpaceToDepthMode::BLOCKS_FIRST, static_cast(blocksize)); + return Ptr(new InfEngineNgraphNode(output_node)); + } +#endif // HAVE_DNN_NGRAPH + +#ifdef HAVE_CUDA + Ptr initCUDA(void *context_, + const std::vector> &inputs, + const std::vector> &outputs) override { + using namespace cv::dnn::cuda4dnn; + auto context = reinterpret_cast(context_); + std::vector perm(permutation.begin(), permutation.end()); + return make_cuda_node(preferableTarget, std::move(context->stream), internal_shape, perm); + } +#endif // HAVE_CUDA + +#ifdef HAVE_CANN + virtual Ptr initCann(const std::vector> &inputs, + const std::vector> &outputs, + const std::vector> &nodes) CV_OVERRIDE { + CV_CheckEQ(inputs.size(), static_cast(1), "DepthToSpace/CANN: only accepts one input wrapper"); + CV_CheckEQ(nodes.size(), static_cast(1), "DepthToSpace/CANN: only accepts one input node"); + + auto input_tensor_wrapper = inputs.front().dynamicCast(); + auto input_tensor_desc = input_tensor_wrapper->getTensorDesc(); + auto input_node = nodes.front().dynamicCast()->getOp(); + + auto node = std::make_shared(name); + + node->set_attr_block_size(blocksize); + node->set_attr_data_format("NCHW"); + + node->set_input_x_by_name(*input_node, input_tensor_wrapper->name.c_str()); + node->update_input_desc_x(*input_tensor_desc); + + auto output_tensor_desc = std::make_shared(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + node->update_output_desc_y(*output_tensor_desc); + + return Ptr(new CannBackendNode(node)); + } +#endif + +#ifdef HAVE_TIMVX + virtual Ptr initTimVX(void* timvx_info_, + const std::vector> &inputs, + const std::vector> &outputs, + bool isLast) CV_OVERRIDE { + auto info = reinterpret_cast(timvx_info_); + CV_Assert(info); + auto timvx_graph = info->getGraph(); + CV_Assert(timvx_graph); + auto graph = timvx_graph->graph; + + auto input_wrapper = inputs.front().dynamicCast(); + int input_wrapper_index = -1; + if (input_wrapper->isTensor()) { + input_wrapper_index = timvx_graph->getTensorIndex(input_wrapper->getTensor()); + if (input_wrapper_index == -1) { + auto tmp = input_wrapper->getMat(); + input_wrapper = std::make_shared(tmp); + } + } + if (!input_wrapper->isTensor() || input_wrapper_index == 1) { + auto input_node_quant = Ptr(new tim::vx::Quantization(tim::vx::QuantType::ASYMMETRIC, 1.0f, 0)); + input_wrapper->createTensor(graph, tim::vx::TensorAttribute::INPUT, input_node_quant); + input_wrapper_index = timvx_graph->addWrapper(input_wrapper); + } + + auto output_wrapper = outputs.front().dynamicCast(); + auto output_node_quant = input_wrapper->getTensorQuantization(); + if (isLast) { + auto shape_type = getShapeTypeFromMat(output_wrapper->getMat()); + output_wrapper->setTensorShape(shape_type); + output_wrapper->createTensor(graph, tim::vx::TensorAttribute::OUTPUT, output_node_quant); + } else { + output_wrapper->createTensor(graph, tim::vx::TensorAttribute::TRANSIENT, output_node_quant); + } + int output_wrapper_index = timvx_graph->addWrapper(output_wrapper); + + std::shared_ptr timvx_node; + timvx_node = graph->CreateOperation(std::vector{blocksize, blocksize}); + std::vector input_wrapper_indices{input_wrapper_index}, output_wrapper_indices{output_wrapper_index}; + return Ptr(new TimVXBackendNode(timvx_graph, timvx_node, input_wrapper_indices, output_wrapper_indices)); + } +#endif + +private: + int blocksize; +}; + +Ptr SpaceToDepthLayer::create(const LayerParams ¶ms) { + return makePtr(params); +} + +}} // namespace cv::dnn diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 6492a25a2809..0fa4605fba85 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -190,7 +190,7 @@ class ONNXImporter void parseDetectionOutput (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseCumSum (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseElementWise (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - void parseDepthToSpace (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseDepthSpaceOps (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseRange (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseScatter (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseTile (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -2923,82 +2923,8 @@ void ONNXImporter::parseElementWise(LayerParams& layerParams, const opencv_onnx: addLayer(layerParams, node_proto); } -void ONNXImporter::parseDepthToSpace(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) -{ - // We parse "DepthToSpace" and "SpaceToDepth" in this function. - opencv_onnx::NodeProto node_proto = node_proto_; - const std::string& layer_type = node_proto.op_type(); - CV_Assert(layer_type == "DepthToSpace" || layer_type == "SpaceToDepth"); - - // Get blocksize - CV_Assert(layerParams.has("blocksize")); - int blocksize = layerParams.get("blocksize"); - CV_Assert(blocksize > 0); - - // Get mode, only for "DepthToSpace" - std::string modeType = layerParams.get("mode", "DCR"); - - MatShape inpShape = outShapes[node_proto.input(0)]; - CV_Assert(inpShape.size() == 4); - int N = inpShape[0], C = inpShape[1], H = inpShape[2], W = inpShape[3]; - - // Implement DepthToSpace and SpaceToDepth by the Reshape and Permute layer. - std::array shape0, perm; - std::array shape1; - - if (layer_type == "DepthToSpace") - { - if (modeType == "DCR") - { - shape0 = {N, blocksize, blocksize, C/(blocksize * blocksize), H, W}; - perm = {0, 3, 4, 1, 5, 2}; - shape1 = {N, C/(blocksize * blocksize), H * blocksize, W * blocksize}; - } - else if (modeType == "CRD") - { - shape0 = {N, C/(blocksize * blocksize), blocksize, blocksize, H, W}; - perm = {0, 1, 4, 2, 5, 3}; - shape1 = {N, C/(blocksize * blocksize), H * blocksize, W * blocksize}; - } - else - CV_Error(Error::StsNotImplemented, "The mode of " + modeType + " in " + layer_type + " Layer is not supported"); - } - else // SpaceToDepth - { - shape0 = {N, C, H/blocksize, blocksize, W/blocksize, blocksize}; - perm = {0, 3, 5, 1, 2, 4}; - shape1 = {N, C * blocksize * blocksize, H/blocksize, W/blocksize}; - } - - // Step1: Reshape - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("dim", DictValue::arrayInt(shape0.data(), shape0.size())); - - opencv_onnx::NodeProto protoReshape; - protoReshape.add_input(node_proto.input(0)); - protoReshape.add_output(reshapeLp.name); - addLayer(reshapeLp, protoReshape); - - // Step2: Transpose - LayerParams permuteLp; - permuteLp.name = layerParams.name + "/permute"; - permuteLp.type = "Permute"; - CV_Assert(layer_id.find(permuteLp.name) == layer_id.end()); - permuteLp.set("order", DictValue::arrayInt(perm.data(), perm.size())); - - opencv_onnx::NodeProto protoPermute; - protoPermute.add_input(reshapeLp.name); - protoPermute.add_output(permuteLp.name); - addLayer(permuteLp, protoPermute); - - // Step3: Reshape - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(shape1.data(), shape1.size())); - - node_proto.set_input(0, permuteLp.name); +void ONNXImporter::parseDepthSpaceOps(LayerParams &layerParams, const opencv_onnx::NodeProto& node_proto) { + CV_CheckTrue(layerParams.has("blocksize"), "blocksize is required but not found"); addLayer(layerParams, node_proto); } @@ -4035,7 +3961,7 @@ void ONNXImporter::buildDispatchMap_ONNX_AI(int opset_version) dispatch["SoftMax"] = dispatch["Softmax"] = dispatch["LogSoftmax"] = &ONNXImporter::parseSoftMax; dispatch["DetectionOutput"] = &ONNXImporter::parseDetectionOutput; dispatch["CumSum"] = &ONNXImporter::parseCumSum; - dispatch["SpaceToDepth"] = dispatch["DepthToSpace"] = &ONNXImporter::parseDepthToSpace; + dispatch["SpaceToDepth"] = dispatch["DepthToSpace"] = &ONNXImporter::parseDepthSpaceOps; dispatch["ScatterElements"] = dispatch["Scatter"] = dispatch["ScatterND"] = &ONNXImporter::parseScatter; dispatch["Tile"] = &ONNXImporter::parseTile; dispatch["LayerNormalization"] = &ONNXImporter::parseLayerNorm; diff --git a/modules/dnn/test/test_int8_layers.cpp b/modules/dnn/test/test_int8_layers.cpp index 9a6bb93162a6..acb1c87d33f2 100644 --- a/modules/dnn/test/test_int8_layers.cpp +++ b/modules/dnn/test/test_int8_layers.cpp @@ -517,6 +517,61 @@ TEST_P(Test_Int8_layers, Eltwise) testLayer("split_max", "ONNX", 0.004, 0.012); } +TEST_P(Test_Int8_layers, DepthSpaceOps) { + auto test_layer_with_onnx_conformance_models = [&](const std::string &model_name, double l1, double lInf) { + std::string model_path = _tf("onnx/conformance/node/test_" + model_name + "/model.onnx"); + auto net = readNet(model_path); + + // load reference inputs and outputs + std::string data_base_path = _tf("onnx/conformance/node/test_" + model_name + "/test_data_set_0"); + Mat input = readTensorFromONNX(data_base_path + "/input_0.pb"); + Mat ref_output = readTensorFromONNX(data_base_path + "/output_0.pb"); + + std::vector input_scales, output_scales; + std::vector input_zeropoints, output_zeropoints; + auto qnet = net.quantize(std::vector{input}, CV_8S, CV_8S, false); + qnet.getInputDetails(input_scales, input_zeropoints); + qnet.getOutputDetails(output_scales, output_zeropoints); + qnet.setPreferableBackend(backend); + qnet.setPreferableTarget(target); + + Mat quantized_input, quantized_output; + input.convertTo(quantized_input, CV_8S, 1.f / input_scales.front(), input_zeropoints.front()); + qnet.setInput(quantized_input); + quantized_output = qnet.forward(); + + Mat output; + quantized_output.convertTo(output, CV_32F, output_scales.front(), -(output_scales.front() * output_zeropoints.front())); + normAssert(ref_output, output, model_name.c_str(), l1, lInf); + }; + + double l1 = default_l1, lInf = default_lInf; + { + l1 = 0.001; lInf = 0.002; + if (backend == DNN_BACKEND_TIMVX) { l1 = 0.001; lInf = 0.002; } + test_layer_with_onnx_conformance_models("spacetodepth", l1, lInf); + } + { + l1 = 0.022; lInf = 0.044; + if (backend == DNN_BACKEND_TIMVX) { l1 = 0.022; lInf = 0.044; } + test_layer_with_onnx_conformance_models("spacetodepth_example", l1, lInf); + } + { + l1 = 0.001; lInf = 0.002; + if (backend == DNN_BACKEND_TIMVX) { l1 = 0.24; lInf = 0.99; } + test_layer_with_onnx_conformance_models("depthtospace_crd_mode", l1, lInf); + } + test_layer_with_onnx_conformance_models("depthtospace_dcr_mode", 0.001, 0.002); + test_layer_with_onnx_conformance_models("depthtospace_example", 0.07, 0.14); + + { + l1 = 0.07; lInf = 0.14; + if (backend == DNN_BACKEND_TIMVX) // diff too huge, l1 = 13.6; lInf = 27.2 + applyTestTag(CV_TEST_TAG_DNN_SKIP_TIMVX); + test_layer_with_onnx_conformance_models("depthtospace_crd_mode_example", l1, lInf); + } +} + INSTANTIATE_TEST_CASE_P(/**/, Test_Int8_layers, dnnBackendsAndTargetsInt8()); class Test_Int8_nets : public DNNTestLayer diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp index 2ee7b5181223..865f4c31b2f8 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -480,10 +480,20 @@ CASE(test_cumsum_2d_negative_axis) // no filter CASE(test_depthtospace_crd_mode) // no filter + if (target == DNN_TARGET_OPENCL) + { + default_l1 = 1e-4; // Expected: (normL1) <= (l1), actual: 9.33057e-05 vs 1e-05 + default_lInf = 2.5e-4; // Expected: (normInf) <= (lInf), actual: 0.000243843 vs 0.0001 + } CASE(test_depthtospace_crd_mode_example) // no filter CASE(test_depthtospace_dcr_mode) // no filter + if (target == DNN_TARGET_OPENCL) + { + default_l1 = 1e-4; // Expected: (normL1) <= (l1), actual: 9.33057e-05 vs 1e-05 + default_lInf = 2.5e-4; // Expected: (normInf) <= (lInf), actual: 0.000243843 vs 0.0001 + } CASE(test_depthtospace_example) // no filter CASE(test_dequantizelinear) diff --git a/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp index fb2376ece881..0b6dab6a9d41 100644 --- a/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp @@ -51,6 +51,9 @@ class GAPI_EXPORTS_W_SIMPLE PyParams { GAPI_WRAP PyParams& cfgDisableMemPattern(); + GAPI_WRAP + PyParams& cfgSessionOptions(const std::map& options); + GBackend backend() const; std::string tag() const; cv::util::any params() const; diff --git a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp index f985b41d71bf..fd0f69a768e4 100644 --- a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp @@ -351,6 +351,7 @@ struct ParamDesc { std::unordered_map > generic_mstd; std::unordered_map generic_norm; + std::map session_options; std::vector execution_providers; bool disable_mem_pattern; }; @@ -634,6 +635,19 @@ template class Params { return *this; } + /** @brief Configures session options for ONNX Runtime. + + This function is used to set various session options for the ONNX Runtime + session by accepting a map of key-value pairs. + + @param options A map of session option to be applied to the ONNX Runtime session. + @return the reference on modified object. + */ + Params& cfgSessionOptions(const std::map& options) { + desc.session_options.insert(options.begin(), options.end()); + return *this; + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::onnx::backend(); } std::string tag() const { return Net::tag(); } @@ -661,7 +675,7 @@ class Params { @param model_path path to model file (.onnx file). */ Params(const std::string& tag, const std::string& model_path) - : desc{model_path, 0u, 0u, {}, {}, {}, {}, {}, {}, {}, {}, {}, true, {}, {}, {}, false }, m_tag(tag) {} + : desc{model_path, 0u, 0u, {}, {}, {}, {}, {}, {}, {}, {}, {}, true, {}, {}, {}, {}, false}, m_tag(tag) {} /** @see onnx::Params::cfgMeanStdDev. */ void cfgMeanStdDev(const std::string &layer, @@ -705,6 +719,11 @@ class Params { desc.disable_mem_pattern = true; } + /** @see onnx::Params::cfgSessionOptions. */ + void cfgSessionOptions(const std::map& options) { + desc.session_options.insert(options.begin(), options.end()); + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::onnx::backend(); } std::string tag() const { return m_tag; } diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp index 6191e9ab05b4..dead5e30c212 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp @@ -2915,7 +2915,7 @@ CV_ALWAYS_INLINE typename std::enable_if::type convertto_simd_nocoeff_impl(const uchar* inx, DST* outx) { - v_uint8 a = vx_load(inx); + v_uint8 a = vx_load_low(inx); v_uint16 res = v_expand_low(a); store_i16(outx, res); diff --git a/modules/gapi/src/backends/onnx/bindings_onnx.cpp b/modules/gapi/src/backends/onnx/bindings_onnx.cpp index 0703f1753dd8..294ad8a3cc21 100644 --- a/modules/gapi/src/backends/onnx/bindings_onnx.cpp +++ b/modules/gapi/src/backends/onnx/bindings_onnx.cpp @@ -57,6 +57,12 @@ cv::gapi::onnx::PyParams::cfgDisableMemPattern() { return *this; } +cv::gapi::onnx::PyParams& +cv::gapi::onnx::PyParams::cfgSessionOptions(const std::map& options) { + m_priv->cfgSessionOptions(options); + return *this; +} + cv::gapi::GBackend cv::gapi::onnx::PyParams::backend() const { return m_priv->backend(); } diff --git a/modules/gapi/src/backends/onnx/gonnxbackend.cpp b/modules/gapi/src/backends/onnx/gonnxbackend.cpp index 92b908da700d..e465a1ccc7ba 100644 --- a/modules/gapi/src/backends/onnx/gonnxbackend.cpp +++ b/modules/gapi/src/backends/onnx/gonnxbackend.cpp @@ -702,6 +702,10 @@ ONNXCompiled::ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp) cv::gimpl::onnx::addExecutionProvider(&session_options, ep); } + for (const auto &option : pp.session_options) { + session_options.AddConfigEntry(option.first.c_str(), option.second.c_str()); + } + if (pp.disable_mem_pattern) { session_options.DisableMemPattern(); } diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index d8a7bb050ab2..0108626dfde0 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -49,6 +49,16 @@ list(REMOVE_ITEM highgui_ext_hdrs "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${n set(OPENCV_HIGHGUI_BUILTIN_BACKEND "") +if(WITH_FRAMEBUFFER AND HAVE_FRAMEBUFFER) + set(OPENCV_HIGHGUI_BUILTIN_BACKEND "FB") + add_definitions(-DHAVE_FRAMEBUFFER) + list(APPEND highgui_srcs ${CMAKE_CURRENT_LIST_DIR}/src/window_framebuffer.cpp) + list(APPEND highgui_hdrs ${CMAKE_CURRENT_LIST_DIR}/src/window_framebuffer.hpp) + if(HAVE_FRAMEBUFFER_XVFB) + add_definitions(-DHAVE_FRAMEBUFFER_XVFB) + endif() +endif() + if(WITH_WAYLAND AND HAVE_WAYLAND) set(OPENCV_HIGHGUI_BUILTIN_BACKEND "Wayland") add_definitions(-DHAVE_WAYLAND) diff --git a/modules/highgui/cmake/detect_framebuffer.cmake b/modules/highgui/cmake/detect_framebuffer.cmake new file mode 100644 index 000000000000..8e292f72acab --- /dev/null +++ b/modules/highgui/cmake/detect_framebuffer.cmake @@ -0,0 +1,14 @@ +# --- FB --- +set(HAVE_FRAMEBUFFER ON) +if(WITH_FRAMEBUFFER_XVFB) + try_compile(HAVE_FRAMEBUFFER_XVFB + "${CMAKE_CURRENT_BINARY_DIR}" + "${OpenCV_SOURCE_DIR}/cmake/checks/framebuffer.cpp") + if(HAVE_FRAMEBUFFER_XVFB) + message(STATUS "Check virtual framebuffer - done") + else() + message(STATUS + "Check virtual framebuffer - failed\n" + "Please install the xorg-x11-proto-devel or x11proto-dev package\n") + endif() +endif() diff --git a/modules/highgui/cmake/init.cmake b/modules/highgui/cmake/init.cmake index 49d4799b308b..f52ce2a4a8e3 100644 --- a/modules/highgui/cmake/init.cmake +++ b/modules/highgui/cmake/init.cmake @@ -39,6 +39,8 @@ endmacro() add_backend("gtk" WITH_GTK) add_backend("win32ui" WITH_WIN32UI) add_backend("wayland" WITH_WAYLAND) +add_backend("framebuffer" WITH_FRAMEBUFFER) + # TODO cocoa # TODO qt # TODO opengl diff --git a/modules/highgui/src/backend.hpp b/modules/highgui/src/backend.hpp index 93d51da11966..49878baefc2d 100644 --- a/modules/highgui/src/backend.hpp +++ b/modules/highgui/src/backend.hpp @@ -127,6 +127,10 @@ std::shared_ptr createUIBackendGTK(); std::shared_ptr createUIBackendQT(); #endif +#ifdef HAVE_FRAMEBUFFER +std::shared_ptr createUIBackendFramebuffer(); +#endif + #endif // BUILD_PLUGIN } // namespace highgui_backend diff --git a/modules/highgui/src/registry.impl.hpp b/modules/highgui/src/registry.impl.hpp index 23f4e9f4e1a0..782738a82063 100644 --- a/modules/highgui/src/registry.impl.hpp +++ b/modules/highgui/src/registry.impl.hpp @@ -44,6 +44,10 @@ std::vector& getBuiltinBackendsInfo() DECLARE_DYNAMIC_BACKEND("GTK2") #endif +#ifdef HAVE_FRAMEBUFFER + DECLARE_STATIC_BACKEND("FB", createUIBackendFramebuffer) +#endif + #if 0 // TODO #ifdef HAVE_QT DECLARE_STATIC_BACKEND("QT", createUIBackendQT) diff --git a/modules/highgui/src/window_framebuffer.cpp b/modules/highgui/src/window_framebuffer.cpp new file mode 100644 index 000000000000..687bf9172411 --- /dev/null +++ b/modules/highgui/src/window_framebuffer.cpp @@ -0,0 +1,798 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" + +#include "window_framebuffer.hpp" + +#include +#include +#ifdef NDEBUG +#define CV_LOG_STRIP_LEVEL CV_LOG_LEVEL_DEBUG + 1 +#else +#define CV_LOG_STRIP_LEVEL CV_LOG_LEVEL_VERBOSE + 1 +#endif +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "opencv2/imgproc.hpp" + +#ifdef HAVE_FRAMEBUFFER_XVFB +#include +#include + +#define C32INT(ptr) ((((unsigned char*)ptr)[0] << 24) | (((unsigned char*)ptr)[1] << 16) | \ + (((unsigned char*)ptr)[2] << 8) | (((unsigned char*)ptr)[3] << 0)) +#endif + + +namespace cv { +namespace highgui_backend { + +std::shared_ptr createUIBackendFramebuffer() +{ + return std::make_shared(); +} + +static std::string& getFBMode() +{ + static std::string fbModeOpenCV = + cv::utils::getConfigurationParameterString("OPENCV_HIGHGUI_FB_MODE", "FB"); + return fbModeOpenCV; +} + +static std::string& getFBFileName() +{ + static std::string fbFileNameFB = + cv::utils::getConfigurationParameterString("FRAMEBUFFER", "/dev/fb0"); + static std::string fbFileNameOpenCV = + cv::utils::getConfigurationParameterString("OPENCV_HIGHGUI_FB_DEVICE", ""); + + if (!fbFileNameOpenCV.empty()) return fbFileNameOpenCV; + return fbFileNameFB; +} + +FramebufferWindow::FramebufferWindow(FramebufferBackend &_backend, int _flags): + backend(_backend), flags(_flags) +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::FramebufferWindow()"); + FB_ID = "FramebufferWindow"; + windowRect = Rect(0,0, backend.getFBWidth(), backend.getFBHeight()); +} + +FramebufferWindow::~FramebufferWindow() +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::~FramebufferWindow()"); +} + +void FramebufferWindow::imshow(InputArray image) +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::imshow(InputArray image)"); + currentImg = image.getMat().clone(); + + CV_LOG_INFO(NULL, "UI: InputArray image: " + << cv::typeToString(image.type()) << " size " << image.size()); + + if (currentImg.empty()) + { + CV_LOG_WARNING(NULL, "UI: image is empty"); + return; + } + CV_CheckEQ(currentImg.dims, 2, "UI: dims != 2"); + + Mat img = image.getMat(); + switch (img.channels()) + { + case 1: + { + Mat tmp; + switch(img.type()) + { + case CV_8U: + tmp = img; + break; + case CV_8S: + cv::convertScaleAbs(img, tmp, 1, 127); + break; + case CV_16S: + cv::convertScaleAbs(img, tmp, 1/255., 127); + break; + case CV_16U: + cv::convertScaleAbs(img, tmp, 1/255.); + break; + case CV_32F: + case CV_64F: // assuming image has values in range [0, 1) + img.convertTo(tmp, CV_8U, 255., 0.); + break; + } + Mat rgb(img.rows, img.cols, CV_8UC3); + cvtColor(tmp, rgb, COLOR_GRAY2RGB); + img = rgb; + } + break; + case 3: + case 4: + { + Mat tmp(img.rows, img.cols, CV_8UC3); + convertToShow(img, tmp, true); + img = tmp; + } + break; + default: + CV_Error(cv::Error::StsBadArg, "Bad image: wrong number of channels"); + } + { + Mat bgra(img.rows, img.cols, CV_8UC4); + cvtColor(img, bgra, COLOR_RGB2BGRA, bgra.channels()); + img = bgra; + } + + int newWidth = windowRect.width; + int newHeight = windowRect.height; + int cntChannel = img.channels(); + cv::Size imgSize = currentImg.size(); + + if (flags & WINDOW_AUTOSIZE) + { + windowRect.width = imgSize.width; + windowRect.height = imgSize.height; + newWidth = windowRect.width; + newHeight = windowRect.height; + } + + if (flags & WINDOW_FREERATIO) + { + newWidth = windowRect.width; + newHeight = windowRect.height; + } + else //WINDOW_KEEPRATIO + { + double aspect_ratio = ((double)img.cols) / img.rows; + newWidth = windowRect.width; + newHeight = (int)(windowRect.width / aspect_ratio); + + if (newHeight > windowRect.height) + { + newWidth = (int)(windowRect.height * aspect_ratio); + newHeight = windowRect.height; + } + } + + if ((newWidth != img.cols) && (newHeight != img.rows)) + { + Mat imResize; + cv::resize(img, imResize, cv::Size(newWidth, newHeight), INTER_LINEAR); + img = imResize; + } + + CV_LOG_INFO(NULL, "UI: Formated image: " + << cv::typeToString(img.type()) << " size " << img.size()); + + if (backend.getMode() == FB_MODE_EMU) + { + CV_LOG_WARNING(NULL, "UI: FramebufferWindow::imshow is used in EMU mode"); + return; + } + + if (backend.getFBPointer() == MAP_FAILED) + { + CV_LOG_ERROR(NULL, "UI: Framebuffer is not mapped"); + return; + } + + int xOffset = backend.getFBXOffset(); + int yOffset = backend.getFBYOffset(); + int fbHeight = backend.getFBHeight(); + int fbWidth = backend.getFBWidth(); + int lineLength = backend.getFBLineLength(); + + int img_start_x; + int img_start_y; + int img_end_x; + int img_end_y; + int fb_start_x; + int fb_start_y; + + if (windowRect.y - yOffset < 0) + { + img_start_y = - (windowRect.y - yOffset); + } + else + { + img_start_y = 0; + } + if (windowRect.x - xOffset < 0) + { + img_start_x = - (windowRect.x - xOffset); + } + else + { + img_start_x = 0; + } + + if (windowRect.y + yOffset + img.rows > fbHeight) + { + img_end_y = fbHeight - windowRect.y - yOffset; + } + else + { + img_end_y = img.rows; + } + if (windowRect.x + xOffset + img.cols > fbWidth) + { + img_end_x = fbWidth - windowRect.x - xOffset; + } + else + { + img_end_x = img.cols; + } + + if (windowRect.y + yOffset >= 0) + { + fb_start_y = windowRect.y + yOffset; + } + else + { + fb_start_y = 0; + } + if (windowRect.x + xOffset >= 0) + { + fb_start_x = windowRect.x + xOffset; + } + else + { + fb_start_x = 0; + } + + for (int y = img_start_y; y < img_end_y; y++) + { + std::memcpy(backend.getFBPointer() + + (fb_start_y + y - img_start_y) * lineLength + fb_start_x * cntChannel, + img.ptr(y) + img_start_x * cntChannel, + (img_end_x - img_start_x) * cntChannel); + } +} + +double FramebufferWindow::getProperty(int /*prop*/) const +{ + CV_LOG_WARNING(NULL, "UI: getProperty (not supported)"); + return 0.0; +} + +bool FramebufferWindow::setProperty(int /*prop*/, double /*value*/) +{ + CV_LOG_WARNING(NULL, "UI: setProperty (not supported)"); + return false; +} + +void FramebufferWindow::resize(int width, int height) +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::resize(int width " + << width <<", height " << height << ")"); + + CV_Assert(width > 0); + CV_Assert(height > 0); + + if (!(flags & WINDOW_AUTOSIZE)) + { + windowRect.width = width; + windowRect.height = height; + + if (!currentImg.empty()) + { + imshow(currentImg); + } + } +} + +void FramebufferWindow::move(int x, int y) +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::move(int x " << x << ", y " << y <<")"); + + windowRect.x = x; + windowRect.y = y; + + if (!currentImg.empty()) + { + imshow(currentImg); + } +} + +Rect FramebufferWindow::getImageRect() const +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::getImageRect()"); + return windowRect; +} + +void FramebufferWindow::setTitle(const std::string& /*title*/) +{ + CV_LOG_WARNING(NULL, "UI: setTitle (not supported)"); +} + +void FramebufferWindow::setMouseCallback(MouseCallback /*onMouse*/, void* /*userdata*/) +{ + CV_LOG_WARNING(NULL, "UI: setMouseCallback (not supported)"); +} + +std::shared_ptr FramebufferWindow::createTrackbar( + const std::string& /*name*/, + int /*count*/, + TrackbarCallback /*onChange*/, + void* /*userdata*/) +{ + CV_LOG_WARNING(NULL, "UI: createTrackbar (not supported)"); + return nullptr; +} + +std::shared_ptr FramebufferWindow::findTrackbar(const std::string& /*name*/) +{ + CV_LOG_WARNING(NULL, "UI: findTrackbar (not supported)"); + return nullptr; +} + +const std::string& FramebufferWindow::getID() const +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::getID()"); + return FB_ID; +} + +bool FramebufferWindow::isActive() const +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::isActive()"); + return true; +} + +void FramebufferWindow::destroy() +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::destroy()"); +} + +int FramebufferBackend::fbOpenAndGetInfo() +{ + std::string fbFileName = getFBFileName(); + CV_LOG_INFO(NULL, "UI: FramebufferWindow::The following is used as a framebuffer file: \n" + << fbFileName); + + int fb_fd = open(fbFileName.c_str(), O_RDWR); + if (fb_fd == -1) + { + CV_LOG_ERROR(NULL, "UI: can't open framebuffer"); + return -1; + } + + if (ioctl(fb_fd, FBIOGET_FSCREENINFO, &fixInfo)) + { + CV_LOG_ERROR(NULL, "UI: can't read fix info for framebuffer"); + return -1; + } + + if (ioctl(fb_fd, FBIOGET_VSCREENINFO, &varInfo)) + { + CV_LOG_ERROR(NULL, "UI: can't read var info for framebuffer"); + return -1; + } + + CV_LOG_INFO(NULL, "UI: framebuffer info: \n" + << " red offset " << varInfo.red.offset << " length " << varInfo.red.length << "\n" + << " green offset " << varInfo.green.offset << " length " << varInfo.green.length << "\n" + << " blue offset " << varInfo.blue.offset << " length " << varInfo.blue.length << "\n" + << "transp offset " << varInfo.transp.offset << " length " <pixmap_format)) != ZPixmap) + { + CV_LOG_ERROR(NULL, "Unsupported pixmap format: " << xwd_header->pixmap_format); + return -1; + } + + if (xwd_header->xoffset != 0) + { + CV_LOG_ERROR(NULL, "UI: Unsupported xoffset value: " << xwd_header->xoffset ); + return -1; + } + + unsigned int r = C32INT(&(xwd_header->red_mask)); + unsigned int g = C32INT(&(xwd_header->green_mask)); + unsigned int b = C32INT(&(xwd_header->blue_mask)); + + fbWidth = C32INT(&(xwd_header->pixmap_width)); + fbHeight = C32INT(&(xwd_header->pixmap_height)); + fbXOffset = 0; + fbYOffset = 0; + fbLineLength = C32INT(&(xwd_header->bytes_per_line)); + fbBitsPerPixel = C32INT(&(xwd_header->bits_per_pixel)); + + CV_LOG_INFO(NULL, "UI: XVFB info: \n" + << " red_mask " << r << "\n" + << " green_mask " << g << "\n" + << " blue_mask " << b << "\n" + << "bits_per_pixel " << fbBitsPerPixel); + + if ((r != 16711680 ) && (g != 65280 ) && (b != 255 ) && + (fbBitsPerPixel != 32)) + { + CV_LOG_ERROR(NULL, "UI: Framebuffer format is not supported " + << "(use BGRA format with bits_per_pixel = 32)"); + return -1; + } + + xvfb_len_header = C32INT(&(xwd_header->header_size)); + xvfb_len_colors = sizeof(XWDColor) * C32INT(&(xwd_header->ncolors)); + xvfb_len_pixmap = C32INT(&(xwd_header->bytes_per_line)) * + C32INT(&(xwd_header->pixmap_height)); + + munmap(xwd_header, sizeof(XWDFileHeader)); + + fbScreenSize = xvfb_len_header + xvfb_len_colors + xvfb_len_pixmap; + xwd_header = (XWDFileHeader*) + mmap(NULL, fbScreenSize, PROT_READ | PROT_WRITE, MAP_SHARED, fb_fd, 0); + + fbPointer = (unsigned char*)xwd_header; + fbPointer_dist = xvfb_len_header + xvfb_len_colors; + +#else + CV_LOG_WARNING(NULL, "UI: To use virtual framebuffer, " + << "compile OpenCV with the WITH_FRAMEBUFFER_XVFB=ON"); +#endif + + return fb_fd; +} + +fb_var_screeninfo &FramebufferBackend::getVarInfo() +{ + return varInfo; +} + +fb_fix_screeninfo &FramebufferBackend::getFixInfo() +{ + return fixInfo; +} + +int FramebufferBackend::getFramebuffrerID() +{ + return fbID; +} + +int FramebufferBackend::getFBWidth() +{ + return fbWidth; +} + +int FramebufferBackend::getFBHeight() +{ + return fbHeight; +} + +int FramebufferBackend::getFBXOffset() +{ + return fbXOffset; +} + +int FramebufferBackend::getFBYOffset() +{ + return fbYOffset; +} + +int FramebufferBackend::getFBBitsPerPixel() +{ + return fbBitsPerPixel; +} + +int FramebufferBackend::getFBLineLength() +{ + return fbLineLength; +} + +unsigned char* FramebufferBackend::getFBPointer() +{ + return fbPointer + fbPointer_dist; +} + +Mat& FramebufferBackend::getBackgroundBuff() +{ + return backgroundBuff; +} + +OpenCVFBMode FramebufferBackend::getMode() +{ + return mode; +} + +FramebufferBackend::FramebufferBackend():mode(FB_MODE_FB), fbPointer_dist(0) +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferWindow::FramebufferBackend()"); + + std::string fbModeStr = getFBMode(); + + if (fbModeStr == "EMU") + { + mode = FB_MODE_EMU; + CV_LOG_WARNING(NULL, "UI: FramebufferWindow is trying to use EMU mode"); + } + if (fbModeStr == "FB") + { + mode = FB_MODE_FB; + CV_LOG_WARNING(NULL, "UI: FramebufferWindow is trying to use FB mode"); + } + if (fbModeStr == "XVFB") + { + mode = FB_MODE_XVFB; + CV_LOG_WARNING(NULL, "UI: FramebufferWindow is trying to use XVFB mode"); + } + + fbID = -1; + if (mode == FB_MODE_FB) + { + fbID = fbOpenAndGetInfo(); + } + if (mode == FB_MODE_XVFB) + { + fbID = XvfbOpenAndGetInfo(); + } + + CV_LOG_INFO(NULL, "UI: FramebufferWindow::fbID " << fbID); + + if (fbID == -1) + { + mode = FB_MODE_EMU; + fbWidth = 640; + fbHeight = 480; + fbXOffset = 0; + fbYOffset = 0; + fbBitsPerPixel = 0; + fbLineLength = 0; + + CV_LOG_WARNING(NULL, "UI: FramebufferWindow is used in EMU mode"); + return; + } + + CV_LOG_INFO(NULL, "UI: Framebuffer's width, height, bits per pix: " + << fbWidth << " " << fbHeight << " " << fbBitsPerPixel); + + CV_LOG_INFO(NULL, "UI: Framebuffer's offsets (x, y), line length: " + << fbXOffset << " " << fbYOffset << " " << fbLineLength); + + backgroundBuff = Mat(fbHeight, fbWidth, CV_8UC4); + int cntChannel = 4; + for (int y = fbYOffset; y < backgroundBuff.rows + fbYOffset; y++) + { + std::memcpy(backgroundBuff.ptr(y - fbYOffset), + getFBPointer() + y * fbLineLength + fbXOffset * cntChannel, + backgroundBuff.cols * cntChannel); + } +} + +FramebufferBackend::~FramebufferBackend() +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferBackend::~FramebufferBackend()"); + if(fbID == -1) return; + + if (fbPointer != MAP_FAILED) + { + int cntChannel = 4; + for (int y = fbYOffset; y < backgroundBuff.rows + fbYOffset; y++) + { + std::memcpy(getFBPointer() + y * fbLineLength + fbXOffset * cntChannel, + backgroundBuff.ptr(y - fbYOffset), + backgroundBuff.cols * cntChannel); + } + + munmap(fbPointer, fbScreenSize); + } + close(fbID); +} + +void FramebufferBackend::destroyAllWindows() { + CV_LOG_DEBUG(NULL, "UI: FramebufferBackend::destroyAllWindows()"); +} + +// namedWindow +std::shared_ptr FramebufferBackend::createWindow( + const std::string& winname, + int flags) +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferBackend::createWindow(" + << winname << ", " << flags << ")"); + return std::make_shared(*this, flags); +} + +void FramebufferBackend::initTermios(int echo, int wait) +{ + tcgetattr(0, &old); + current = old; + current.c_lflag &= ~ICANON; + current.c_lflag &= ~ISIG; + current.c_cc[VMIN] = wait; + if (echo) + { + current.c_lflag |= ECHO; + } + else + { + current.c_lflag &= ~ECHO; + } + tcsetattr(0, TCSANOW, ¤t); +} + +void FramebufferBackend::resetTermios(void) +{ + tcsetattr(0, TCSANOW, &old); +} + +int FramebufferBackend::getch_(int echo, int wait) +{ + int ch; + initTermios(echo, wait); + ch = getchar(); + if (ch < 0) + { + rewind(stdin); + } + resetTermios(); + return ch; +} + +bool FramebufferBackend::kbhit() +{ + int byteswaiting = 0; + initTermios(0, 1); + if (ioctl(0, FIONREAD, &byteswaiting) < 0) + { + CV_LOG_ERROR(NULL, "UI: Framebuffer ERR byteswaiting" ); + } + resetTermios(); + + return byteswaiting > 0; +} + +int FramebufferBackend::waitKeyEx(int delay) +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferBackend::waitKeyEx(int delay = " << delay << ")"); + + int code = -1; + + if (delay <= 0) + { + int ch = getch_(0, 1); + CV_LOG_INFO(NULL, "UI: FramebufferBackend::getch_() take value = " << (int)ch); + code = ch; + + while ((ch = getch_(0, 0)) >= 0) + { + CV_LOG_INFO(NULL, "UI: FramebufferBackend::getch_() take value = " + << (int)ch << " (additional code on )"); + code = ch; + } + } + else + { + bool f_kbhit = false; + while (!(f_kbhit = kbhit()) && (delay > 0)) + { + delay -= 1; + usleep(1000); + } + if (f_kbhit) + { + CV_LOG_INFO(NULL, "UI: FramebufferBackend kbhit is True "); + + int ch = getch_(0, 1); + CV_LOG_INFO(NULL, "UI: FramebufferBackend::getch_() take value = " << (int)ch); + code = ch; + + while ((ch = getch_(0, 0)) >= 0) + { + CV_LOG_INFO(NULL, "UI: FramebufferBackend::getch_() take value = " + << (int)ch << " (additional code on )"); + code = ch; + } + } + } + + CV_LOG_INFO(NULL, "UI: FramebufferBackend::waitKeyEx() result code = " << code); + return code; +} + +int FramebufferBackend::pollKey() +{ + CV_LOG_DEBUG(NULL, "UI: FramebufferBackend::pollKey()"); + int code = -1; + bool f_kbhit = false; + f_kbhit = kbhit(); + + if (f_kbhit) + { + CV_LOG_INFO(NULL, "UI: FramebufferBackend kbhit is True "); + + int ch = getch_(0, 1); + CV_LOG_INFO(NULL, "UI: FramebufferBackend::getch_() take value = " << (int)ch); + code = ch; + + while ((ch = getch_(0, 0)) >= 0) + { + CV_LOG_INFO(NULL, "UI: FramebufferBackend::getch_() take value = " + << (int)ch << " (additional code on )"); + code = ch; + } + } + + return code; +} + +const std::string FramebufferBackend::getName() const +{ + return "FB"; +} + +}} // cv::highgui_backend:: diff --git a/modules/highgui/src/window_framebuffer.hpp b/modules/highgui/src/window_framebuffer.hpp new file mode 100644 index 000000000000..e7bf50dff661 --- /dev/null +++ b/modules/highgui/src/window_framebuffer.hpp @@ -0,0 +1,135 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_HIGHGUI_WINDOWS_FRAMEBUFFER_HPP +#define OPENCV_HIGHGUI_WINDOWS_FRAMEBUFFER_HPP + +#include "backend.hpp" + +#include +#include + +#include + +namespace cv { +namespace highgui_backend { + +enum OpenCVFBMode{ + FB_MODE_EMU, + FB_MODE_FB, + FB_MODE_XVFB +}; + +class FramebufferBackend; +class FramebufferWindow : public UIWindow +{ + FramebufferBackend &backend; + std::string FB_ID; + Rect windowRect; + + int flags; + Mat currentImg; + +public: + FramebufferWindow(FramebufferBackend &backend, int flags); + virtual ~FramebufferWindow(); + + virtual void imshow(InputArray image) override; + + virtual double getProperty(int prop) const override; + virtual bool setProperty(int prop, double value) override; + + virtual void resize(int width, int height) override; + virtual void move(int x, int y) override; + + virtual Rect getImageRect() const override; + + virtual void setTitle(const std::string& title) override; + + virtual void setMouseCallback(MouseCallback onMouse, void* userdata /*= 0*/) override; + + virtual std::shared_ptr createTrackbar( + const std::string& name, + int count, + TrackbarCallback onChange /*= 0*/, + void* userdata /*= 0*/ + ) override; + + virtual std::shared_ptr findTrackbar(const std::string& name) override; + + virtual const std::string& getID() const override; + + virtual bool isActive() const override; + + virtual void destroy() override; +}; // FramebufferWindow + +class FramebufferBackend: public UIBackend +{ + OpenCVFBMode mode; + + struct termios old, current; + + void initTermios(int echo, int wait); + void resetTermios(void); + int getch_(int echo, int wait); + bool kbhit(); + + fb_var_screeninfo varInfo; + fb_fix_screeninfo fixInfo; + int fbWidth; + int fbHeight; + int fbXOffset; + int fbYOffset; + int fbBitsPerPixel; + int fbLineLength; + long int fbScreenSize; + unsigned char* fbPointer; + unsigned int fbPointer_dist; + Mat backgroundBuff; + + int fbOpenAndGetInfo(); + int fbID; + + unsigned int xvfb_len_header; + unsigned int xvfb_len_colors; + unsigned int xvfb_len_pixmap; + int XvfbOpenAndGetInfo(); + +public: + + fb_var_screeninfo &getVarInfo(); + fb_fix_screeninfo &getFixInfo(); + int getFramebuffrerID(); + int getFBWidth(); + int getFBHeight(); + int getFBXOffset(); + int getFBYOffset(); + int getFBBitsPerPixel(); + int getFBLineLength(); + unsigned char* getFBPointer(); + Mat& getBackgroundBuff(); + OpenCVFBMode getMode(); + + FramebufferBackend(); + + virtual ~FramebufferBackend(); + + virtual void destroyAllWindows()override; + + // namedWindow + virtual std::shared_ptr createWindow( + const std::string& winname, + int flags + )override; + + virtual int waitKeyEx(int delay /*= 0*/)override; + virtual int pollKey() override; + + virtual const std::string getName() const override; +}; + +}} // cv::highgui_backend:: + +#endif diff --git a/modules/highgui/src/window_w32.cpp b/modules/highgui/src/window_w32.cpp index 44efcd583511..2bb0e68bcb53 100644 --- a/modules/highgui/src/window_w32.cpp +++ b/modules/highgui/src/window_w32.cpp @@ -676,7 +676,7 @@ void cvSetPropTopmost_W32(const char* name, const bool topmost) static bool setPropTopmost_(CvWindow& window, bool topmost) { - HWND flag = topmost ? HWND_TOPMOST : HWND_TOP; + HWND flag = topmost ? HWND_TOPMOST : HWND_NOTOPMOST; BOOL success = SetWindowPos(window.frame, flag, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE); if (!success) diff --git a/modules/highgui/test/test_gui.cpp b/modules/highgui/test/test_gui.cpp index 5b72545faf5f..b8093e07ecec 100644 --- a/modules/highgui/test/test_gui.cpp +++ b/modules/highgui/test/test_gui.cpp @@ -148,7 +148,8 @@ static void Foo(int, void* counter) && !defined HAVE_WIN32UI \ && !defined HAVE_WAYLAND \ ) \ - || defined(__APPLE__) // test fails on Mac (cocoa) + || defined(__APPLE__) /* test fails on Mac (cocoa) */ \ + || defined HAVE_FRAMEBUFFER /* trackbar is not supported */ TEST(Highgui_GUI, DISABLED_trackbar_unsafe) #else TEST(Highgui_GUI, trackbar_unsafe) @@ -188,7 +189,8 @@ void testTrackbarCallback(int pos, void* param) && !defined HAVE_WIN32UI \ && !defined HAVE_WAYLAND \ ) \ - || defined(__APPLE__) // test fails on Mac (cocoa) + || defined(__APPLE__) /* test fails on Mac (cocoa) */ \ + || defined HAVE_FRAMEBUFFER /* trackbar is not supported */ TEST(Highgui_GUI, DISABLED_trackbar) #else TEST(Highgui_GUI, trackbar) diff --git a/modules/imgcodecs/src/grfmt_hdr.cpp b/modules/imgcodecs/src/grfmt_hdr.cpp index 3a18b5bbebc4..c9fec94aa329 100644 --- a/modules/imgcodecs/src/grfmt_hdr.cpp +++ b/modules/imgcodecs/src/grfmt_hdr.cpp @@ -59,6 +59,9 @@ HdrDecoder::HdrDecoder() HdrDecoder::~HdrDecoder() { + if(file) { + fclose(file); + } } size_t HdrDecoder::signatureLength() const diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index 85c8f6346a3a..a5b4116834e0 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -4269,7 +4269,7 @@ Examples of how intersectConvexConvex works When false, no intersection is found. If the polygons share a side or the vertex of one polygon lies on an edge of the other, they are not considered nested and an intersection will be found regardless of the value of handleNested. -@returns Absolute value of area of intersecting polygon +@returns Area of intersecting polygon. May be negative, if algorithm has not converged, e.g. non-convex input. @note intersectConvexConvex doesn't confirm that both polygons are convex and will return invalid results if they aren't. */ diff --git a/modules/imgproc/perf/perf_bilateral.cpp b/modules/imgproc/perf/perf_bilateral.cpp index ed6a741e0865..38f2a503d34e 100644 --- a/modules/imgproc/perf/perf_bilateral.cpp +++ b/modules/imgproc/perf/perf_bilateral.cpp @@ -7,23 +7,25 @@ namespace opencv_test { CV_ENUM(Mat_Type, CV_8UC1, CV_8UC3, CV_32FC1, CV_32FC3) -typedef TestBaseWithParam< tuple > TestBilateralFilter; +typedef TestBaseWithParam< tuple > TestBilateralFilter; PERF_TEST_P( TestBilateralFilter, BilateralFilter, Combine( Values( szVGA, sz1080p ), // image size Values( 3, 5 ), // d - Mat_Type::all() // image type + Mat_Type::all(), // image type + Values(1., 5.) ) ) { Size sz; int d, type; - const double sigmaColor = 1., sigmaSpace = 1.; + double sigmaColor, sigmaSpace; sz = get<0>(GetParam()); d = get<1>(GetParam()); type = get<2>(GetParam()); + sigmaColor = sigmaSpace = get<3>(GetParam()); Mat src(sz, type); Mat dst(sz, type); diff --git a/modules/imgproc/src/color_hsv.simd.hpp b/modules/imgproc/src/color_hsv.simd.hpp index bef9497760db..c450d609e55d 100644 --- a/modules/imgproc/src/color_hsv.simd.hpp +++ b/modules/imgproc/src/color_hsv.simd.hpp @@ -130,7 +130,7 @@ struct RGB2HSV_b // sdiv = sdiv_table[v] v_int32 sdiv0, sdiv1, sdiv2, sdiv3;; - v_uint16 vd0, vd1, vd2; + v_uint16 vd0, vd1; v_expand(v, vd0, vd1); v_int32 vq0, vq1, vq2, vq3; v_expand(v_reinterpret_as_s16(vd0), vq0, vq1); @@ -150,7 +150,7 @@ struct RGB2HSV_b // hdiv = hdiv_table[diff] v_int32 hdiv0, hdiv1, hdiv2, hdiv3; - v_uint16 diffd0, diffd1, diffd2; + v_uint16 diffd0, diffd1; v_expand(diff, diffd0, diffd1); v_int32 diffq0, diffq1, diffq2, diffq3; v_expand(v_reinterpret_as_s16(diffd0), diffq0, diffq1); diff --git a/modules/imgproc/src/demosaicing.cpp b/modules/imgproc/src/demosaicing.cpp index 2bfa705b8abc..dd285cab516e 100644 --- a/modules/imgproc/src/demosaicing.cpp +++ b/modules/imgproc/src/demosaicing.cpp @@ -593,10 +593,10 @@ class Bayer2Gray_Invoker : public ParallelLoopBody { public: - Bayer2Gray_Invoker(const Mat& _srcmat, Mat& _dstmat, int _start_with_green, bool _brow, + Bayer2Gray_Invoker(const Mat& _srcmat, Mat& _dstmat, int _start_with_green, const Size& _size, int _bcoeff, int _rcoeff) : ParallelLoopBody(), srcmat(_srcmat), dstmat(_dstmat), Start_with_green(_start_with_green), - Brow(_brow), size(_size), Bcoeff(_bcoeff), Rcoeff(_rcoeff) + size(_size), Bcoeff(_bcoeff), Rcoeff(_rcoeff) { } @@ -612,13 +612,11 @@ class Bayer2Gray_Invoker : int dst_step = (int)(dstmat.step/sizeof(T)); int bcoeff = Bcoeff, rcoeff = Rcoeff; int start_with_green = Start_with_green; - bool brow = Brow; dst0 += dst_step + 1; if (range.start % 2) { - brow = !brow; std::swap(bcoeff, rcoeff); start_with_green = !start_with_green; } @@ -680,7 +678,6 @@ class Bayer2Gray_Invoker : dst0[-1] = dst0[0]; dst0[size.width] = dst0[size.width-1]; - brow = !brow; std::swap(bcoeff, rcoeff); start_with_green = !start_with_green; } @@ -690,7 +687,6 @@ class Bayer2Gray_Invoker : Mat srcmat; Mat dstmat; int Start_with_green; - bool Brow; Size size; int Bcoeff, Rcoeff; }; @@ -704,11 +700,9 @@ static void Bayer2Gray_( const Mat& srcmat, Mat& dstmat, int code ) Size size = srcmat.size(); int bcoeff = B2Y, rcoeff = R2Y; int start_with_green = code == COLOR_BayerGB2GRAY || code == COLOR_BayerGR2GRAY; - bool brow = true; if( code != COLOR_BayerBG2GRAY && code != COLOR_BayerGB2GRAY ) { - brow = false; std::swap(bcoeff, rcoeff); } size.height -= 2; @@ -718,7 +712,7 @@ static void Bayer2Gray_( const Mat& srcmat, Mat& dstmat, int code ) { Range range(0, size.height); Bayer2Gray_Invoker invoker(srcmat, dstmat, - start_with_green, brow, size, bcoeff, rcoeff); + start_with_green, size, bcoeff, rcoeff); parallel_for_(range, invoker, dstmat.total()/static_cast(1<<16)); } diff --git a/modules/imgproc/src/geometry.cpp b/modules/imgproc/src/geometry.cpp index f457bb77063e..eb6757c4f0ca 100644 --- a/modules/imgproc/src/geometry.cpp +++ b/modules/imgproc/src/geometry.cpp @@ -318,9 +318,12 @@ static void addSharedSeg( Point2f p, Point2f q, Point2f*& result ) *result++ = q; } - +// Note: The function and subroutings use direct pointer arithmetics instead of arrays indexing. +// Each loop iteration may push to result array up to 3 times. +// It means that we need +3 spare result elements against result_size +// to catch agorithmic overflow and prevent actual result array overflow. static int intersectConvexConvex_( const Point2f* P, int n, const Point2f* Q, int m, - Point2f* result, float* _area ) + Point2f* result, int result_size, float* _area ) { Point2f* result0 = result; // P has n vertices, Q has m vertices. @@ -398,7 +401,7 @@ static int intersectConvexConvex_( const Point2f* P, int n, const Point2f* Q, in } // Quit when both adv. indices have cycled, or one has cycled twice. } - while ( ((aa < n) || (ba < m)) && (aa < 2*n) && (ba < 2*m) ); + while ( ((aa < n) || (ba < m)) && (aa < 2*n) && (ba < 2*m) && ((int)(result - result0) <= result_size) ); // Deal with special cases: not implemented. if( inflag == Unknown ) @@ -407,10 +410,16 @@ static int intersectConvexConvex_( const Point2f* P, int n, const Point2f* Q, in // ... } - int i, nr = (int)(result - result0); + int nr = (int)(result - result0); + if (nr > result_size) + { + *_area = -1.f; + return -1; + } + double area = 0; Point2f prev = result0[nr-1]; - for( i = 1; i < nr; i++ ) + for(int i = 1; i < nr; i++ ) { result0[i-1] = result0[i]; area += (double)prev.x*result0[i].y - (double)prev.y*result0[i].x; @@ -445,9 +454,11 @@ float cv::intersectConvexConvex( InputArray _p1, InputArray _p2, OutputArray _p1 return 0.f; } - AutoBuffer _result(n*2 + m*2 + 1); - Point2f *fp1 = _result.data(), *fp2 = fp1 + n; + AutoBuffer _result(n + m + n+m+1+3); + Point2f* fp1 = _result.data(); + Point2f* fp2 = fp1 + n; Point2f* result = fp2 + m; + int orientation = 0; for( int k = 1; k <= 2; k++ ) @@ -476,7 +487,15 @@ float cv::intersectConvexConvex( InputArray _p1, InputArray _p2, OutputArray _p1 } float area = 0.f; - int nr = intersectConvexConvex_(fp1, n, fp2, m, result, &area); + int nr = intersectConvexConvex_(fp1, n, fp2, m, result, n+m+1, &area); + + if (nr < 0) + { + // The algorithm did not converge, e.g. some of inputs is not convex + _p12.release(); + return -1.f; + } + if( nr == 0 ) { if( !handleNested ) diff --git a/modules/imgproc/test/test_bilateral_filter.cpp b/modules/imgproc/test/test_bilateral_filter.cpp index badd879a8abd..0f92836808fc 100644 --- a/modules/imgproc/test/test_bilateral_filter.cpp +++ b/modules/imgproc/test/test_bilateral_filter.cpp @@ -243,7 +243,7 @@ namespace opencv_test { namespace { rng.fill(_src, RNG::UNIFORM, 0, 256); - _sigma_color = _sigma_space = 1.; + _sigma_color = _sigma_space = rng.uniform(0., 10.); return 1; } diff --git a/modules/imgproc/test/test_histograms.cpp b/modules/imgproc/test/test_histograms.cpp index aebc3ff7c514..2d0c4bb15caf 100644 --- a/modules/imgproc/test/test_histograms.cpp +++ b/modules/imgproc/test/test_histograms.cpp @@ -142,5 +142,74 @@ TEST(Imgproc_Hist_Calc, IPP_ranges_with_nonequal_exponent_21595) ASSERT_EQ(histogram_u.at(2), 4.f) << "1 not counts correctly, res: " << histogram_u.at(2); } +////////////////////////////////////////// equalizeHist() ///////////////////////////////////////// + +void equalizeHistReference(const Mat& src, Mat& dst) +{ + std::vector hist(256, 0); + for (int y = 0; y < src.rows; y++) + { + const uchar* srow = src.ptr(y); + for (int x = 0; x < src.cols; x++) + { + hist[srow[x]]++; + } + } + + int first = 0; + while (!hist[first]) ++first; + + int total = (int)src.total(); + if (hist[first] == total) + { + dst.setTo(first); + return; + } + + std::vector lut(256); + lut[first] = 0; + float scale = (255.f)/(total - hist[first]); + + int sum = 0; + for (int i = first + 1; i < 256; ++i) + { + sum += hist[i]; + lut[i] = saturate_cast(sum * scale); + } + + cv::LUT(src, lut, dst); +} + +typedef ::testing::TestWithParam> Imgproc_Equalize_Hist; + +TEST_P(Imgproc_Equalize_Hist, accuracy) +{ + auto p = GetParam(); + cv::Size size = std::get<0>(p); + int idx = std::get<1>(p); + + RNG &rng = cvtest::TS::ptr()->get_rng(); + rng.state += idx; + + cv::Mat src(size, CV_8U); + cvtest::randUni(rng, src, Scalar::all(0), Scalar::all(255)); + + cv::Mat dst, gold; + + equalizeHistReference(src, gold); + + cv::equalizeHist(src, dst); + + ASSERT_EQ(CV_8UC1, dst.type()); + ASSERT_EQ(gold.size(), dst.size()); + + EXPECT_MAT_NEAR(dst, gold, 1); + EXPECT_MAT_N_DIFF(dst, gold, 0.05 * size.area()); // The 5% range could be accomodated to HAL +} + +INSTANTIATE_TEST_CASE_P(Imgproc_Hist, Imgproc_Equalize_Hist, ::testing::Combine( + ::testing::Values(cv::Size(123, 321), cv::Size(256, 256), cv::Size(1024, 768)), + ::testing::Range(0, 10))); + }} // namespace /* End Of File */ diff --git a/modules/imgproc/test/test_intersectconvexconvex.cpp b/modules/imgproc/test/test_intersectconvexconvex.cpp index fa25f3d531e3..00e3674f48d0 100644 --- a/modules/imgproc/test/test_intersectconvexconvex.cpp +++ b/modules/imgproc/test/test_intersectconvexconvex.cpp @@ -255,6 +255,42 @@ TEST(Imgproc_IntersectConvexConvex, intersection_4) EXPECT_NEAR(area, 391500, std::numeric_limits::epsilon()); } +// The inputs are not convex and cuased buffer overflow +// See https://github.com/opencv/opencv/issues/25259 +TEST(Imgproc_IntersectConvexConvex, not_convex) +{ + std::vector convex1 = { + { 46.077175f , 228.66121f }, { 5.428622f , 250.05899f }, {207.51741f , 109.645676f }, + {175.94789f , 32.6566f }, {217.4915f , 252.66176f }, {187.09386f , 6.3988557f}, + { 52.20488f , 69.266205f }, { 38.188286f , 134.48068f }, {246.4742f , 31.41043f }, + {178.97946f , 169.52287f }, {103.40764f , 153.30397f }, {160.67746f , 17.166115f }, + {152.44255f , 135.35f }, {197.03804f , 193.04782f }, {248.28397f , 56.821487f }, + { 10.907227f , 82.55291f }, {109.67949f , 70.7405f }, { 58.96842f , 150.132f }, + {150.7613f , 129.54753f }, {254.98463f , 228.21748f }, {139.02563f , 193.89336f }, + { 84.79946f , 162.25363f }, { 39.83567f , 44.626484f }, {107.034996f , 209.38887f }, + { 67.61073f , 17.119232f }, {208.8617f , 33.67367f }, {182.65207f , 8.291072f }, + { 72.89319f , 42.51845f }, {202.4902f , 123.97209f }, { 79.945076f , 140.99268f }, + {225.8952f , 66.226326f }, { 34.08404f , 219.2208f }, {243.1221f , 60.95162f } + }; + std::vector convex2 = { + {144.33624f , 247.15732f }, { 5.656847f , 17.461054f }, {230.54338f , 2.0446582f}, + {143.0578f , 215.27856f }, {250.44626f , 82.54287f }, { 0.3846766f, 11.101262f }, + { 70.81022f , 17.243904f }, { 77.18812f , 75.760666f }, {190.34933f , 234.30962f }, + {230.10204f , 133.67998f }, { 58.903755f , 252.96451f }, {213.57228f , 155.7058f }, + {190.80992f , 212.90802f }, {203.4356f , 36.55016f }, { 32.276424f , 2.5646307f}, + { 39.73823f , 87.23782f }, {112.46902f , 101.81753f }, { 58.154305f , 238.40395f }, + {187.01064f , 96.24343f }, { 44.42692f , 10.573529f }, {118.76949f , 233.35114f }, + { 86.26109f , 120.93148f }, {217.94751f , 130.5933f }, {148.2687f , 68.56015f }, + {187.44174f , 214.32857f }, {247.19875f , 180.8494f }, { 17.986013f , 61.451443f }, + {254.74344f , 204.71747f }, {211.92726f , 132.0139f }, { 51.36624f , 116.63085f }, + { 83.80044f , 124.20074f }, {122.125854f , 25.182402f }, { 39.08164f , 180.08517f } + }; + std::vector intersection; + + float area = cv::intersectConvexConvex(convex1, convex2, intersection, false); + EXPECT_TRUE(intersection.empty()); + EXPECT_LE(area, 0.f); +} } // namespace } // opencv_test diff --git a/modules/js/test/init_cv.js b/modules/js/test/init_cv.js index d05370740784..e6c1d1c16abe 100644 --- a/modules/js/test/init_cv.js +++ b/modules/js/test/init_cv.js @@ -2,13 +2,18 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. -if (cv instanceof Promise) { - QUnit.test("init_cv", (assert) => { +QUnit.test("init_cv", (assert) => { + if (cv instanceof Promise) { const done = assert.async(); - assert.ok(true); cv.then((ready_cv) => { cv = ready_cv; done(); }); - }); -} + } else if (cv.getBuildInformation === undefined) { + const done = assert.async(); + cv['onRuntimeInitialized'] = () => { + done(); + } + } + assert.ok(true); +}); diff --git a/modules/video/include/opencv2/video/tracking.hpp b/modules/video/include/opencv2/video/tracking.hpp index 7176cbd7b88f..21bdbb39cadc 100644 --- a/modules/video/include/opencv2/video/tracking.hpp +++ b/modules/video/include/opencv2/video/tracking.hpp @@ -879,6 +879,7 @@ class CV_EXPORTS_W TrackerVit : public Tracker CV_PROP_RW int target; CV_PROP_RW Scalar meanvalue; CV_PROP_RW Scalar stdvalue; + CV_PROP_RW float tracking_score_threshold; }; /** @brief Constructor diff --git a/modules/video/src/tracking/tracker_vit.cpp b/modules/video/src/tracking/tracker_vit.cpp index bef42dbb4d7e..1dfdde2a1626 100644 --- a/modules/video/src/tracking/tracker_vit.cpp +++ b/modules/video/src/tracking/tracker_vit.cpp @@ -24,8 +24,8 @@ TrackerVit::~TrackerVit() TrackerVit::Params::Params() { net = "vitTracker.onnx"; - meanvalue = Scalar{0.485, 0.456, 0.406}; - stdvalue = Scalar{0.229, 0.224, 0.225}; + meanvalue = Scalar{0.485, 0.456, 0.406}; // normalized mean (already divided by 255) + stdvalue = Scalar{0.229, 0.224, 0.225}; // normalized std (already divided by 255) #ifdef HAVE_OPENCV_DNN backend = dnn::DNN_BACKEND_DEFAULT; target = dnn::DNN_TARGET_CPU; @@ -33,6 +33,7 @@ TrackerVit::Params::Params() backend = -1; // invalid value target = -1; // invalid value #endif + tracking_score_threshold = 0.20f; // safe threshold to filter out black frames } #ifdef HAVE_OPENCV_DNN @@ -48,6 +49,9 @@ class TrackerVitImpl : public TrackerVit net.setPreferableBackend(params.backend); net.setPreferableTarget(params.target); + + i2bp.mean = params.meanvalue * 255.0; + i2bp.scalefactor = (1.0 / params.stdvalue) * (1 / 255.0); } void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; @@ -58,6 +62,7 @@ class TrackerVitImpl : public TrackerVit float tracking_score; TrackerVit::Params params; + dnn::Image2BlobParams i2bp; protected: @@ -69,10 +74,9 @@ class TrackerVitImpl : public TrackerVit Mat hanningWindow; dnn::Net net; - Mat image; }; -static void crop_image(const Mat& src, Mat& dst, Rect box, int factor) +static int crop_image(const Mat& src, Mat& dst, Rect box, int factor) { int x = box.x, y = box.y, w = box.width, h = box.height; int crop_sz = cvCeil(sqrt(w * h) * factor); @@ -90,21 +94,16 @@ static void crop_image(const Mat& src, Mat& dst, Rect box, int factor) Rect roi(x1 + x1_pad, y1 + y1_pad, x2 - x2_pad - x1 - x1_pad, y2 - y2_pad - y1 - y1_pad); Mat im_crop = src(roi); copyMakeBorder(im_crop, dst, y1_pad, y2_pad, x1_pad, x2_pad, BORDER_CONSTANT); + + return crop_sz; } void TrackerVitImpl::preprocess(const Mat& src, Mat& dst, Size size) { - Mat mean = Mat(size, CV_32FC3, params.meanvalue); - Mat std = Mat(size, CV_32FC3, params.stdvalue); - mean = dnn::blobFromImage(mean, 1.0, Size(), Scalar(), false); - std = dnn::blobFromImage(std, 1.0, Size(), Scalar(), false); - Mat img; resize(src, img, size); - dst = dnn::blobFromImage(img, 1.0, Size(), Scalar(), false); - dst /= 255; - dst = (dst - mean) / std; + dst = dnn::blobFromImageWithParams(img, i2bp); } static Mat hann1d(int sz, bool centered = true) { @@ -141,22 +140,21 @@ static Mat hann2d(Size size, bool centered = true) { return hanningWindow; } -static Rect returnfromcrop(float x, float y, float w, float h, Rect res_Last) +static void updateLastRect(float cx, float cy, float w, float h, int crop_size, Rect &rect_last) { - int cropwindowwh = 4 * cvFloor(sqrt(res_Last.width * res_Last.height)); - int x0 = res_Last.x + (res_Last.width - cropwindowwh) / 2; - int y0 = res_Last.y + (res_Last.height - cropwindowwh) / 2; - Rect finalres; - finalres.x = cvFloor(x * cropwindowwh + x0); - finalres.y = cvFloor(y * cropwindowwh + y0); - finalres.width = cvFloor(w * cropwindowwh); - finalres.height = cvFloor(h * cropwindowwh); - return finalres; + int x0 = rect_last.x + (rect_last.width - crop_size) / 2; + int y0 = rect_last.y + (rect_last.height - crop_size) / 2; + + float x1 = cx - w / 2, y1 = cy - h / 2; + rect_last.x = cvFloor(x1 * crop_size + x0); + rect_last.y = cvFloor(y1 * crop_size + y0); + rect_last.width = cvFloor(w * crop_size); + rect_last.height = cvFloor(h * crop_size); } void TrackerVitImpl::init(InputArray image_, const Rect &boundingBox_) { - image = image_.getMat().clone(); + Mat image = image_.getMat(); Mat crop; crop_image(image, crop, boundingBox_, 2); Mat blob; @@ -169,9 +167,9 @@ void TrackerVitImpl::init(InputArray image_, const Rect &boundingBox_) bool TrackerVitImpl::update(InputArray image_, Rect &boundingBoxRes) { - image = image_.getMat().clone(); + Mat image = image_.getMat(); Mat crop; - crop_image(image, crop, rect_last, 4); + int crop_size = crop_image(image, crop, rect_last, 4); // crop: [crop_size, crop_size] Mat blob; preprocess(crop, blob, searchSize); net.setInput(blob, "search"); @@ -191,15 +189,18 @@ bool TrackerVitImpl::update(InputArray image_, Rect &boundingBoxRes) minMaxLoc(conf_map, nullptr, &maxVal, nullptr, &maxLoc); tracking_score = static_cast(maxVal); - float cx = (maxLoc.x + offset_map.at(0, maxLoc.y, maxLoc.x)) / 16; - float cy = (maxLoc.y + offset_map.at(1, maxLoc.y, maxLoc.x)) / 16; - float w = size_map.at(0, maxLoc.y, maxLoc.x); - float h = size_map.at(1, maxLoc.y, maxLoc.x); - - Rect finalres = returnfromcrop(cx - w / 2, cy - h / 2, w, h, rect_last); - rect_last = finalres; - boundingBoxRes = finalres; - return true; + if (tracking_score >= params.tracking_score_threshold) { + float cx = (maxLoc.x + offset_map.at(0, maxLoc.y, maxLoc.x)) / 16; + float cy = (maxLoc.y + offset_map.at(1, maxLoc.y, maxLoc.x)) / 16; + float w = size_map.at(0, maxLoc.y, maxLoc.x); + float h = size_map.at(1, maxLoc.y, maxLoc.x); + + updateLastRect(cx, cy, w, h, crop_size, rect_last); + boundingBoxRes = rect_last; + return true; + } else { + return false; + } } float TrackerVitImpl::getTrackingScore() diff --git a/modules/video/test/test_trackers.cpp b/modules/video/test/test_trackers.cpp index 33ed92d971c9..da695a3434b0 100644 --- a/modules/video/test/test_trackers.cpp +++ b/modules/video/test/test_trackers.cpp @@ -143,9 +143,7 @@ TEST(vittrack, accuracy_vittrack) cv::TrackerVit::Params params; params.net = model; cv::Ptr tracker = TrackerVit::create(params); - // NOTE: Test threshold was reduced from 0.67 (libjpeg-turbo) to 0.66 (libjpeg 9f), - // becase libjpeg and libjpeg-turbo produce slightly different images - checkTrackingAccuracy(tracker, 0.66); + checkTrackingAccuracy(tracker, 0.64); } }} // namespace opencv_test:: diff --git a/modules/videoio/CMakeLists.txt b/modules/videoio/CMakeLists.txt index 776c1bb84d74..c8a03e72ae54 100644 --- a/modules/videoio/CMakeLists.txt +++ b/modules/videoio/CMakeLists.txt @@ -49,6 +49,9 @@ endif() # Removing WinRT API headers by default list(REMOVE_ITEM videoio_ext_hdrs "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/cap_winrt.hpp") +# Remove iOS API header by default +list(REMOVE_ITEM videoio_ext_hdrs "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/cap_ios.h") + if(DEFINED WINRT AND NOT DEFINED ENABLE_WINRT_MODE_NATIVE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /ZW") endif() @@ -221,6 +224,7 @@ if(TARGET ocv.3rdparty.cap_ios) ${CMAKE_CURRENT_LIST_DIR}/src/cap_ios_photo_camera.mm ${CMAKE_CURRENT_LIST_DIR}/src/cap_ios_video_camera.mm) list(APPEND tgts ocv.3rdparty.cap_ios) + list(APPEND videoio_ext_hdrs "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/cap_ios.h") endif() if(TARGET ocv.3rdparty.android_mediandk) diff --git a/modules/videoio/src/cap_gstreamer.cpp b/modules/videoio/src/cap_gstreamer.cpp index 41e98794b96a..cda125334faf 100644 --- a/modules/videoio/src/cap_gstreamer.cpp +++ b/modules/videoio/src/cap_gstreamer.cpp @@ -1618,7 +1618,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam { //do not emit signals: all calls will be synchronous and blocking gst_app_sink_set_emit_signals (GST_APP_SINK(sink.get()), FALSE); - caps.attach(gst_caps_from_string("video/x-raw, format=(string){BGR, GRAY8}; video/x-bayer,format=(string){rggb,bggr,grbg,gbrg}; image/jpeg")); + caps.attach(gst_caps_from_string("video/x-raw, format=(string){BGR}; video/x-raw, format=(string){BGRx, BGRA}; video/x-bayer,format=(string){rggb,bggr,grbg,gbrg}; video/x-raw, format=(string){GRAY8}; image/jpeg")); } if (audioStream >= 0) { diff --git a/platforms/android/build_sdk.py b/platforms/android/build_sdk.py index 7ba3fb8e0d08..83f40904d51c 100755 --- a/platforms/android/build_sdk.py +++ b/platforms/android/build_sdk.py @@ -267,10 +267,14 @@ def build_library(self, abi, do_install, no_media_ndk): cmake_vars['WITH_ANDROID_MEDIANDK'] = "OFF" if self.hwasan and "arm64" in abi.name: + cmake_vars['OPENCV_ENABLE_MEMORY_SANITIZER'] = "ON" hwasan_flags = "-fno-omit-frame-pointer -fsanitize=hwaddress" - cmake_vars['CMAKE_CXX_FLAGS_DEBUG'] = hwasan_flags - cmake_vars['CMAKE_C_FLAGS_DEBUG'] = hwasan_flags - cmake_vars['CMAKE_LINKER_FLAGS_DEBUG'] = hwasan_flags + for s in ['OPENCV_EXTRA_C_FLAGS', 'OPENCV_EXTRA_CXX_FLAGS', 'OPENCV_EXTRA_EXE_LINKER_FLAGS', + 'OPENCV_EXTRA_SHARED_LINKER_FLAGS', 'OPENCV_EXTRA_MODULE_LINKER_FLAGS']: + if s in cmake_vars.keys(): + cmake_vars[s] = cmake_vars[s] + ' ' + hwasan_flags + else: + cmake_vars[s] = hwasan_flags cmake_vars.update(abi.cmake_vars) diff --git a/samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp b/samples/cpp/tutorial_code/videoio/openni_orbbec_astra/openni_orbbec_astra.cpp similarity index 100% rename from samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp rename to samples/cpp/tutorial_code/videoio/openni_orbbec_astra/openni_orbbec_astra.cpp diff --git a/samples/dnn/download_models.py b/samples/dnn/download_models.py index 520f70edcdfd..546a6beab122 100644 --- a/samples/dnn/download_models.py +++ b/samples/dnn/download_models.py @@ -3,7 +3,6 @@ ''' from __future__ import print_function import os -import cv2 import sys import yaml import argparse diff --git a/samples/dnn/models.yml b/samples/dnn/models.yml index 96550403c642..5577c9ca4e0c 100644 --- a/samples/dnn/models.yml +++ b/samples/dnn/models.yml @@ -62,6 +62,20 @@ yolov8m: background_label_id: 0 sample: "yolo_detector" +yolov8l: + load_info: + url: "https://github.com/CVHub520/X-AnyLabeling/releases/download/v0.1.0/yolov8l.onnx" + sha1: "462df53ca3a85d110bf6be7fc2e2bb1277124395" + model: "yolov8l.onnx" + mean: 0.0 + scale: 0.00392 + width: 640 + height: 640 + rgb: true + classes: "object_detection_classes_yolo.txt" + background_label_id: 0 + sample: "yolo_detector" + # YOLO4 object detection family from Darknet (https://github.com/AlexeyAB/darknet) # YOLO object detection family from Darknet (https://pjreddie.com/darknet/yolo/) # Might be used for all YOLOv2, TinyYolov2, YOLOv3, YOLOv4 and TinyYolov4 diff --git a/samples/dnn/vit_tracker.cpp b/samples/dnn/vit_tracker.cpp index 02e5cea83fd5..32974713a68d 100644 --- a/samples/dnn/vit_tracker.cpp +++ b/samples/dnn/vit_tracker.cpp @@ -16,6 +16,7 @@ const char *keys = "{ help h | | Print help message }" "{ input i | | Full path to input video folder, the specific camera index. (empty for camera 0) }" "{ net | vitTracker.onnx | Path to onnx model of vitTracker.onnx}" + "{ tracking_score_threshold t | 0.3 | Tracking score threshold. If a bbox of score >= 0.3, it is considered as found }" "{ backend | 0 | Choose one of computation backends: " "0: automatically (by default), " "1: Halide language (http://halide-lang.org/), " @@ -49,6 +50,7 @@ int run(int argc, char** argv) std::string net = parser.get("net"); int backend = parser.get("backend"); int target = parser.get("target"); + float tracking_score_threshold = parser.get("tracking_score_threshold"); Ptr tracker; try @@ -57,6 +59,7 @@ int run(int argc, char** argv) params.net = samples::findFile(net); params.backend = backend; params.target = target; + params.tracking_score_threshold = tracking_score_threshold; tracker = TrackerVit::create(params); } catch (const cv::Exception& ee) @@ -108,6 +111,11 @@ int run(int argc, char** argv) Rect selectRect = selectROI(winName, image_select); std::cout << "ROI=" << selectRect << std::endl; + if (selectRect.empty()) + { + std::cerr << "Invalid ROI!" << std::endl; + return 2; + } tracker->init(image, selectRect); @@ -130,30 +138,29 @@ int run(int argc, char** argv) float score = tracker->getTrackingScore(); - std::cout << "frame " << count << - ": predicted score=" << score << - " rect=" << rect << - " time=" << tickMeter.getTimeMilli() << "ms" << - std::endl; + std::cout << "frame " << count; + if (ok) { + std::cout << ": predicted score=" << score << + "\trect=" << rect << + "\ttime=" << tickMeter.getTimeMilli() << "ms" << std::endl; - Mat render_image = image.clone(); - - if (ok) - { - rectangle(render_image, rect, Scalar(0, 255, 0), 2); + rectangle(image, rect, Scalar(0, 255, 0), 2); std::string timeLabel = format("Inference time: %.2f ms", tickMeter.getTimeMilli()); std::string scoreLabel = format("Score: %f", score); - putText(render_image, timeLabel, Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); - putText(render_image, scoreLabel, Point(0, 35), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + putText(image, timeLabel, Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + putText(image, scoreLabel, Point(0, 35), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + } else { + std::cout << ": target lost" << std::endl; + putText(image, "Target lost", Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 0, 255)); } - imshow(winName, render_image); + imshow(winName, image); tickMeter.reset(); int c = waitKey(1); - if (c == 27 /*ESC*/) + if (c == 27 /*ESC*/ || c == 'q' || c == 'Q') break; }