Thanks to visit codestin.com
Credit goes to github.com

Skip to content

G-API OV backend requires cv::MediaFrame #24938

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Apr 24, 2024
Merged

G-API OV backend requires cv::MediaFrame #24938

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8248a39
Initial changes to support MediaFrame
DariaMityagina Feb 13, 2024
7d30147
Initial changes to support MediaFrame - review
DariaMityagina Feb 14, 2024
0f5220b
Initial changes to support MediaFrame - review - remove cfgColorFormat
DariaMityagina Feb 14, 2024
38c0ee2
Initial changes to support MediaFrame - review - refactor and add tests
DariaMityagina Feb 20, 2024
7d58666
Initial changes to support MediaFrame - review - refactor/clean up/fi…
DariaMityagina Apr 15, 2024
655777a
Initial changes to support MediaFrame - review - test image
DariaMityagina Apr 16, 2024
2f249a8
Initial changes to support MediaFrame - review - p2
DariaMityagina Apr 23, 2024
584ace9
Merge branch 'opencv:4.x' into icv/dm/add-media-frame-support-to-govb…
DariaMityagina Apr 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 135 additions & 31 deletions modules/gapi/src/backends/ov/govbackend.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ static int toCV(const ov::element::Type &type) {
static void copyFromOV(const ov::Tensor &tensor, cv::Mat &mat) {
const auto total = mat.total() * mat.channels();
if (toCV(tensor.get_element_type()) != mat.depth() ||
tensor.get_size() != total ) {
tensor.get_size() != total) {
std::stringstream ss;
ss << "Failed to copy data from ov::Tensor to cv::Mat."
<< " Data type or number of elements mismatch."
Expand All @@ -151,6 +151,30 @@ static void copyFromOV(const ov::Tensor &tensor, cv::Mat &mat) {
}
}

cv::Mat wrapOV(const cv::MediaFrame::View& view,
const cv::GFrameDesc& desc) {
cv::Mat out;
switch (desc.fmt) {
case cv::MediaFormat::BGR: {
out = cv::Mat(desc.size, CV_8UC3, view.ptr[0], view.stride[0]);
return out;
}
case cv::MediaFormat::NV12: {
auto y_plane = cv::Mat(desc.size, CV_8UC1, view.ptr[0], view.stride[0]);
auto uv_plane = cv::Mat(desc.size / 2, CV_8UC2, view.ptr[1], view.stride[1]);
cvtColorTwoPlane(y_plane, uv_plane, out, cv::COLOR_YUV2BGR_NV12);
return out;
}
case cv::MediaFormat::GRAY: {
out = cv::Mat(desc.size, CV_8UC1, view.ptr[0], view.stride[0]);
return out;
}
default:
GAPI_Error("OV Backend: Unsupported media format");
}
return out;
}

static void copyToOV(const cv::Mat &mat, ov::Tensor &tensor) {
// TODO: Ideally there should be check that mat and tensor
// dimensions are compatible.
Expand All @@ -177,6 +201,12 @@ static void copyToOV(const cv::Mat &mat, ov::Tensor &tensor) {
}
}

static void copyToOV(const cv::MediaFrame &frame, ov::Tensor &tensor) {
const auto view = cv::MediaFrame::View(frame.access(cv::MediaFrame::Access::R));
auto matFromFrame = wrapOV(view, frame.desc());
copyToOV(matFromFrame, tensor);
}

std::vector<int> cv::gapi::ov::util::to_ocv(const ::ov::Shape &shape) {
return toCV(shape);
}
Expand Down Expand Up @@ -269,8 +299,9 @@ class OVCallContext
}

// Syntax sugar
cv::GShape inShape(std::size_t input) const;
const cv::Mat& inMat (std::size_t input) const;
cv::GShape inShape (std::size_t input) const;
const cv::Mat& inMat (std::size_t input) const;
const cv::MediaFrame& inFrame (std::size_t input) const;

cv::GRunArgP output (std::size_t idx);
cv::Mat& outMatR(std::size_t idx);
Expand Down Expand Up @@ -355,6 +386,10 @@ const cv::Mat& OVCallContext::inMat(std::size_t input) const {
return inArg<cv::Mat>(input);
}

const cv::MediaFrame& OVCallContext::inFrame(std::size_t input) const {
return inArg<cv::MediaFrame>(input);
}

cv::Mat& OVCallContext::outMatR(std::size_t idx) {
return *cv::util::get<cv::Mat*>(m_results.at(idx));
}
Expand Down Expand Up @@ -394,6 +429,8 @@ cv::GArg OVCallContext::packArg(const cv::GArg &arg) {
// (and constructed by either bindIn/Out or resetInternal)
case cv::GShape::GOPAQUE: return cv::GArg(m_res.slot<cv::detail::OpaqueRef>().at(ref.id));

case cv::GShape::GFRAME: return cv::GArg(m_res.slot<cv::MediaFrame>()[ref.id]);

default:
cv::util::throw_error(std::logic_error("Unsupported GShape type"));
break;
Expand Down Expand Up @@ -655,6 +692,19 @@ void PostOutputsList::operator()(::ov::InferRequest &infer_request,
}
}

static void copyToOV(std::shared_ptr<OVCallContext> ctx, uint32_t input_idx, ov::Tensor &tensor) {
switch (ctx->inShape(input_idx)) {
case cv::GShape::GMAT:
copyToOV(ctx->inMat(input_idx), tensor);
break;
case cv::GShape::GFRAME:
copyToOV(ctx->inFrame(input_idx), tensor);
break;
default:
GAPI_Assert("Unsupported input shape for OV backend");
}
}

namespace cv {
namespace gimpl {
namespace ov {
Expand Down Expand Up @@ -730,6 +780,37 @@ static cv::Mat preprocess(const cv::Mat &in_mat,
return out;
}

// NB: This function is used to preprocess input image
// for InferROI, InferList, InferList2 kernels.
cv::Mat preprocess(MediaFrame::View& view,
const cv::GFrameDesc& desc,
const cv::Rect& roi,
const ::ov::Shape &model_shape) {
return preprocess(wrapOV(view, desc), roi, model_shape);
}

static void preprocess_and_copy(std::shared_ptr<OVCallContext> ctx,
uint32_t input_idx,
const cv::Rect &roi,
const ::ov::Shape &model_shape,
::ov::Tensor& tensor) {
switch (ctx->inShape(input_idx)) {
case cv::GShape::GMAT: {
auto roi_mat = preprocess(ctx->inMat(input_idx), roi, model_shape);
copyToOV(roi_mat, tensor);
break;
}
case cv::GShape::GFRAME: {
auto currentFrame = ctx->inFrame(input_idx);
auto view = cv::MediaFrame::View(currentFrame.access(cv::MediaFrame::Access::R));
auto roi_mat = preprocess(view, currentFrame.desc(), roi, model_shape);
copyToOV(roi_mat, tensor);
}
default:
GAPI_Assert("Unsupported input shape for OV backend");
}
}

static bool isImage(const cv::GMatDesc &desc,
const ::ov::Shape &model_shape) {
return (model_shape.size() == 4u) &&
Expand All @@ -739,6 +820,16 @@ static bool isImage(const cv::GMatDesc &desc,
(desc.depth == CV_8U);
}

static bool isImage(const cv::GMetaArg &meta,
const ::ov::Shape &shape) {
if (cv::util::holds_alternative<GFrameDesc>(meta)) {
return true;
}
GAPI_Assert(cv::util::holds_alternative<GMatDesc>(meta));
auto matdesc = cv::util::get<GMatDesc>(meta);
return isImage(matdesc, shape);
}

class PrePostProcWrapper {
public:
PrePostProcWrapper(std::shared_ptr<::ov::Model> &model,
Expand Down Expand Up @@ -821,9 +912,8 @@ class PrePostProcWrapper {
void cfgPreProcessing(const std::string &input_name,
const cv::GMetaArg &input_meta,
const bool disable_img_resize = false) {
GAPI_Assert(cv::util::holds_alternative<cv::GMatDesc>(input_meta));
const auto &matdesc = cv::util::get<cv::GMatDesc>(input_meta);

GAPI_Assert(cv::util::holds_alternative<cv::GMatDesc>(input_meta) ||
cv::util::holds_alternative<cv::GFrameDesc>(input_meta));
const auto explicit_in_tensor_layout = lookUp(m_input_tensor_layout, input_name);
const auto explicit_in_model_layout = lookUp(m_input_model_layout, input_name);
const auto explicit_resize = lookUp(m_interpolation, input_name);
Expand All @@ -838,24 +928,35 @@ class PrePostProcWrapper {
const auto &input_shape = m_model->input(input_name).get_shape();
auto &input_info = m_ppp.input(input_name);

m_ppp.input(input_name).tensor().set_element_type(toOV(matdesc.depth));
if (isImage(matdesc, input_shape)) {
auto isMat = cv::util::holds_alternative<cv::GMatDesc>(input_meta);
auto prec = isMat ? cv::util::get<cv::GMatDesc>(input_meta).depth : CV_8U;
m_ppp.input(input_name).tensor().set_element_type(toOV(prec));

const auto &matdesc = isMat ? cv::util::get<cv::GMatDesc>(input_meta) : cv::GMatDesc();
const auto &framedesc = !isMat ? cv::util::get<cv::GFrameDesc>(input_meta) : cv::GFrameDesc();
if (isImage(input_meta, input_shape)) {
// NB: Image case - all necessary preprocessng is configured automatically.
GAPI_LOG_DEBUG(NULL, "OV Backend: Input: \"" << input_name << "\" is image.");
if (explicit_in_tensor_layout &&
*explicit_in_tensor_layout != "NHWC") {
if (explicit_in_tensor_layout && *explicit_in_tensor_layout != "NHWC") {
std::stringstream desc_str;
if (isMat) {
desc_str << matdesc;
} else {
desc_str << framedesc;
}
std::stringstream ss;
ss << "OV Backend: Provided tensor layout " << *explicit_in_tensor_layout
<< " is not compatible with input data " << matdesc << " for layer \""
<< input_name << "\". Expecting NHWC";
<< " is not compatible with input data " << desc_str.str() << " for layer \""
<< input_name << "\". Expecting NHWC";
util::throw_error(std::logic_error(ss.str()));
} else {
input_info.tensor().set_layout(::ov::Layout("NHWC"));
}

if (!disable_img_resize) {
input_info.tensor().set_spatial_static_shape(matdesc.size.height,
matdesc.size.width);
const auto size = isMat ? cv::util::get<cv::GMatDesc>(input_meta).size : cv::util::get<cv::GFrameDesc>(input_meta).size;
input_info.tensor().set_spatial_static_shape(size.height,
size.width);
// NB: Even though resize is automatically configured
// user have an opportunity to specify the interpolation algorithm.
auto interp = explicit_resize
Expand All @@ -877,8 +978,8 @@ class PrePostProcWrapper {
if (!explicit_in_tensor_layout && model_layout.empty()) {
std::stringstream ss;
ss << "Resize for input layer: " << input_name
<< "can't be configured."
<< " Failed to extract H and W positions from layout.";
<< "can't be configured."
<< " Failed to extract H and W positions from layout.";
util::throw_error(std::logic_error(ss.str()));
} else {
const auto layout = explicit_in_tensor_layout
Expand Down Expand Up @@ -982,7 +1083,6 @@ struct Infer: public cv::detail::KernelTag {
ade::util::toRange(in_metas))) {
const auto &input_name = std::get<0>(it);
const auto &mm = std::get<1>(it);

ppp.cfgLayouts(input_name);
ppp.cfgPreProcessing(input_name, mm);
ppp.cfgScaleMean(input_name, mm);
Expand Down Expand Up @@ -1025,7 +1125,7 @@ struct Infer: public cv::detail::KernelTag {
auto input_tensor = infer_request.get_tensor(input_name);
// TODO: In some cases wrapping existing data pointer
// might be faster than copy. Make it a strategy.
copyToOV(ctx->inMat(i), input_tensor);
copyToOV(ctx, i, input_tensor);
}
},
std::bind(PostOutputs, _1, _2, ctx)
Expand Down Expand Up @@ -1054,13 +1154,13 @@ struct InferROI: public cv::detail::KernelTag {

const auto &input_name = uu.params.input_names.at(0);
const auto &mm = in_metas.at(1u);
GAPI_Assert(cv::util::holds_alternative<cv::GMatDesc>(mm));
const auto &matdesc = cv::util::get<cv::GMatDesc>(mm);

GAPI_Assert(cv::util::holds_alternative<cv::GMatDesc>(mm) ||
cv::util::holds_alternative<cv::GFrameDesc>(mm));
const bool is_model = cv::util::holds_alternative<ParamDesc::Model>(uu.params.kind);
const auto &input_shape = is_model ? uu.model->input(input_name).get_shape()
: uu.compiled_model.input(input_name).get_shape();
if (!isImage(matdesc, input_shape)) {

if (!isImage(mm, input_shape)) {
util::throw_error(std::runtime_error(
"OV Backend: InferROI supports only image as the 1th argument"));
}
Expand Down Expand Up @@ -1111,8 +1211,7 @@ struct InferROI: public cv::detail::KernelTag {
auto input_tensor = infer_request.get_tensor(input_name);
const auto &shape = input_tensor.get_shape();
const auto &roi = ctx->inArg<cv::detail::OpaqueRef>(0).rref<cv::Rect>();
const auto roi_mat = preprocess(ctx->inMat(1), roi, shape);
copyToOV(roi_mat, input_tensor);
preprocess_and_copy(ctx, 1, roi, shape, input_tensor);
},
std::bind(PostOutputs, _1, _2, ctx)
}
Expand Down Expand Up @@ -1147,11 +1246,11 @@ struct InferList: public cv::detail::KernelTag {
size_t idx = 1u;
for (auto &&input_name : uu.params.input_names) {
const auto &mm = in_metas[idx++];
GAPI_Assert(cv::util::holds_alternative<cv::GMatDesc>(mm));
const auto &matdesc = cv::util::get<cv::GMatDesc>(mm);
GAPI_Assert(cv::util::holds_alternative<cv::GMatDesc>(mm) ||
cv::util::holds_alternative<cv::GFrameDesc>(mm));
const auto &input_shape = uu.model->input(input_name).get_shape();

if (!isImage(matdesc, input_shape)) {
if (!isImage(mm, input_shape)) {
util::throw_error(std::runtime_error(
"OV Backend: Only image is supported"
" as the " + std::to_string(idx) + "th argument for InferList"));
Expand Down Expand Up @@ -1208,8 +1307,7 @@ struct InferList: public cv::detail::KernelTag {
const auto &input_name = ctx->uu.params.input_names[0];
auto input_tensor = infer_request.get_tensor(input_name);
const auto &shape = input_tensor.get_shape();
const auto roi_mat = preprocess(ctx->inMat(1), rc, shape);
copyToOV(roi_mat, input_tensor);
preprocess_and_copy(ctx, 1, rc, shape, input_tensor);
},
std::bind(callback, std::placeholders::_1, std::placeholders::_2, pos)
}
Expand Down Expand Up @@ -1247,12 +1345,18 @@ struct InferList2: public cv::detail::KernelTag {

const auto &input_name_0 = uu.params.input_names.front();
const auto &mm_0 = in_metas[0u];
const auto &matdesc = cv::util::get<cv::GMatDesc>(mm_0);

if (!(cv::util::holds_alternative<cv::GMatDesc>(mm_0) ||
cv::util::holds_alternative<cv::GFrameDesc>(mm_0))) {
util::throw_error(std::runtime_error(
"OV Backend: Unsupported input meta"
" for 0th argument in OV backend"));
}

const bool is_model = cv::util::holds_alternative<ParamDesc::Model>(uu.params.kind);
const auto &input_shape = is_model ? uu.model->input(input_name_0).get_shape()
: uu.compiled_model.input(input_name_0).get_shape();
if (!isImage(matdesc, input_shape)) {
if (!isImage(mm_0, input_shape)) {
util::throw_error(std::runtime_error(
"OV Backend: InferList2 supports only image as the 0th argument"));
}
Expand Down
Loading