From b9ca972d659191845f8391a7d7ca316a1ce3e16a Mon Sep 17 00:00:00 2001 From: lwasylow Date: Mon, 23 Apr 2018 19:44:15 +0100 Subject: [PATCH 01/24] Initial check in of unordered comparision --- .../data_values/ut_compound_data_helper.pkb | 54 ++++++++++- .../data_values/ut_compound_data_helper.pks | 5 + .../data_values/ut_compound_data_value.tpb | 91 +++++++++++++++++-- .../data_values/ut_compound_data_value.tps | 8 +- .../data_values/ut_data_value.tpb | 4 +- .../data_values/ut_data_value.tps | 4 +- .../data_values/ut_data_value_anydata.tps | 2 +- .../data_values/ut_data_value_refcursor.tpb | 31 ++++++- .../data_values/ut_data_value_refcursor.tps | 6 +- source/expectations/matchers/ut_equal.tpb | 21 ++++- source/expectations/matchers/ut_equal.tps | 5 + .../expectations/ut_expectation_compound.tpb | 18 ++++ .../expectations/ut_expectation_compound.tps | 5 +- 13 files changed, 227 insertions(+), 27 deletions(-) diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index a69741aa8..ed9e397d2 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -217,12 +217,10 @@ create or replace package body ut_compound_data_helper is from ut_compound_data_tmp ucd where ucd.data_id = :other_guid and ucd.item_no in (select i.item_no from diff_info i) - )act on exp.item_no = act.item_no where exp.item_no is null or act.item_no is null - order by 1, 2 -]' + order by 1, 2]' bulk collect into l_results using a_diff_id, a_max_rows, a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, @@ -231,6 +229,56 @@ create or replace package body ut_compound_data_helper is return l_results; end; + function get_rows_diff_unordered( + a_expected_dataset_guid raw, a_actual_dataset_guid raw, a_diff_id raw, + a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2 + ) return tt_row_diffs is + l_column_filter varchar2(32767); + l_results tt_row_diffs; + begin + l_column_filter := get_columns_filter(a_exclude_xpath,a_include_xpath); + execute immediate q'[ + select + coalesce(exp.duplicate_no, act.duplicate_no) duplicate_no, + case when exp.row_hash is null then 'Actual:' else 'Expected:' end diffed_type, + case when exp.row_hash is null then + xmlserialize(content act.row_data no indent) + else + xmlserialize(content exp.row_data no indent) + end diffed_row + from (select ucd.*, row_number() over(partition by row_hash order by row_hash) duplicate_no + from (select ucd.column_value row_data, + dbms_crypto.hash( value(ucd).getclobval(),3) row_hash + from (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter + from ut_compound_data_tmp ucd + where ucd.data_id = :self_guid + ) r, + table( xmlsequence( extract(r.item_data,'/*') ) ) ucd + ) ucd + ) exp + full outer join + (select ucd.*, row_number() over(partition by row_hash order by row_hash) duplicate_no + from (select ucd.column_value row_data, + dbms_crypto.hash( value(ucd).getclobval(),3/*HASH_SH1*/) row_hash + from (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter + from ut_compound_data_tmp ucd + where ucd.data_id = :other_guid + ) r, + table( xmlsequence( extract(r.item_data,'/*') ) ) ucd + ) ucd + ) act + on exp.row_hash = act.row_hash + and exp.duplicate_no = act.duplicate_no + where exp.row_hash is null or act.row_hash is null]' + bulk collect into l_results + using a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, + a_exclude_xpath, a_include_xpath, a_actual_dataset_guid; + + --execute immediate 'create table test as select * from ut_compound_data_tmp'; + return l_results; + + end; + function get_hash(a_data raw, a_hash_type binary_integer := dbms_crypto.hash_sh1) return t_hash is begin return dbms_crypto.hash(a_data, a_hash_type); diff --git a/source/expectations/data_values/ut_compound_data_helper.pks b/source/expectations/data_values/ut_compound_data_helper.pks index d49db160c..ae9b2a044 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pks +++ b/source/expectations/data_values/ut_compound_data_helper.pks @@ -52,6 +52,11 @@ create or replace package ut_compound_data_helper authid definer is a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2 ) return tt_row_diffs; + function get_rows_diff_unordered( + a_expected_dataset_guid raw, a_actual_dataset_guid raw, a_diff_id raw, + a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2 + ) return tt_row_diffs; + subtype t_hash is raw(128); function get_hash(a_data raw, a_hash_type binary_integer := dbms_crypto.hash_sh1) return t_hash; diff --git a/source/expectations/data_values/ut_compound_data_value.tpb b/source/expectations/data_values/ut_compound_data_value.tpb index b7c43d2d8..8a5fb2ab3 100644 --- a/source/expectations/data_values/ut_compound_data_value.tpb +++ b/source/expectations/data_values/ut_compound_data_value.tpb @@ -71,17 +71,61 @@ create or replace type body ut_compound_data_value as return l_result_string; end; - overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2 ) return varchar2 is + overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean := false ) return varchar2 is l_result clob; l_result_string varchar2(32767); begin - l_result := get_data_diff(a_other, a_exclude_xpath, a_include_xpath); + l_result := get_data_diff(a_other, a_exclude_xpath, a_include_xpath, a_unordered); l_result_string := ut_utils.to_string(l_result,null); dbms_lob.freetemporary(l_result); return l_result_string; end; - member function get_data_diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2 ) return clob is + member function get_data_diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean ) return clob is + c_max_rows constant integer := 20; + l_result clob; + l_results ut_utils.t_clob_tab := ut_utils.t_clob_tab(); + l_message varchar2(32767); + l_ut_owner varchar2(250) := ut_utils.ut_owner; + l_diff_row_count integer; + l_actual ut_compound_data_value; + l_diff_id ut_compound_data_helper.t_hash; + l_row_diffs ut_compound_data_helper.tt_row_diffs; + begin + if not a_other is of (ut_compound_data_value) then + raise value_error; + end if; + l_actual := treat(a_other as ut_compound_data_value); + + dbms_lob.createtemporary(l_result,true); + + --diff rows and row elements + l_diff_id := ut_compound_data_helper.get_hash(self.data_id||l_actual.data_id); + -- First tell how many rows are different + execute immediate 'select unordered_cnt from ' || l_ut_owner || '.ut_compound_data_diff_tmp where diff_id = :diff_id' into l_diff_row_count using l_diff_id; + + if l_diff_row_count > 0 then + l_row_diffs := ut_compound_data_helper.get_rows_diff_unordered( + self.data_id, l_actual.data_id, l_diff_id, c_max_rows, a_exclude_xpath, a_include_xpath + ); + l_message := chr(10) + ||'Rows: [ ' || l_diff_row_count ||' differences' + || case when l_diff_row_count > c_max_rows and l_row_diffs.count > 0 then ', showing first '||c_max_rows end + ||' ]' || chr(10) + || case when l_row_diffs.count = 0 + then ' All rows are different as the columns are not matching.' end; + ut_utils.append_to_clob( l_result, l_message ); + for i in 1 .. l_row_diffs.count loop + l_results.extend; + l_results(l_results.last) := + ' Row Appeared '||l_row_diffs(i).rn||' - '||rpad(l_row_diffs(i).diff_type,10)||l_row_diffs(i).diffed_row; + end loop; + ut_utils.append_to_clob(l_result,l_results); + end if; + return l_result; + end; + + member function get_data_diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2 ) return clob is c_max_rows constant integer := 20; l_result clob; l_results ut_utils.t_clob_tab := ut_utils.t_clob_tab(); @@ -105,9 +149,9 @@ create or replace type body ut_compound_data_value as execute immediate 'select count(*) from ' || l_ut_owner || '.ut_compound_data_diff_tmp where diff_id = :diff_id' into l_diff_row_count using l_diff_id; if l_diff_row_count > 0 then - l_row_diffs := ut_compound_data_helper.get_rows_diff( - self.data_id, l_actual.data_id, l_diff_id, c_max_rows, a_exclude_xpath, a_include_xpath - ); + l_row_diffs := ut_compound_data_helper.get_rows_diff( + self.data_id, l_actual.data_id, l_diff_id, c_max_rows, a_exclude_xpath, a_include_xpath + ); l_message := chr(10) ||'Rows: [ ' || l_diff_row_count ||' differences' || case when l_diff_row_count > c_max_rows and l_row_diffs.count > 0 then ', showing first '||c_max_rows end @@ -171,7 +215,6 @@ create or replace type body ut_compound_data_value as ') != 0' using in l_diff_id, a_exclude_xpath, a_include_xpath, self.data_id, a_exclude_xpath, a_include_xpath, l_other.data_id, l_xml_data_fmt, l_xml_data_fmt; - --result is OK only if both are same if sql%rowcount = 0 and self.elements_count = l_other.elements_count then l_result := 0; @@ -181,5 +224,39 @@ create or replace type body ut_compound_data_value as return l_result; end; +member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean ) return integer is + l_other ut_compound_data_value; + l_ut_owner varchar2(250) := ut_utils.ut_owner; + l_column_filter varchar2(32767); + l_diff_id ut_compound_data_helper.t_hash; + l_result integer; + l_row_diffs ut_compound_data_helper.tt_row_diffs; + c_max_rows constant integer := 20; + begin + if not a_other is of (ut_compound_data_value) then + raise value_error; + end if; + + l_other := treat(a_other as ut_compound_data_value); + + l_diff_id := ut_compound_data_helper.get_hash(self.data_id||l_other.data_id); + l_column_filter := ut_compound_data_helper.get_columns_filter(a_exclude_xpath, a_include_xpath); + + -- Find differences + l_row_diffs := ut_compound_data_helper.get_rows_diff_unordered( + self.data_id, l_other.data_id, l_diff_id, c_max_rows, a_exclude_xpath, a_include_xpath + ); + + execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id, unordered_cnt ) + values (:diff_id, nvl(:unordered_cnt,0))' using l_diff_id, l_row_diffs.count; + --result is OK only if both are same + if l_row_diffs.count = 0 and self.elements_count = l_other.elements_count then + l_result := 0; + else + l_result := 1; + end if; + return l_result; + end; + end; / diff --git a/source/expectations/data_values/ut_compound_data_value.tps b/source/expectations/data_values/ut_compound_data_value.tps index e06848a1e..d2345c01e 100644 --- a/source/expectations/data_values/ut_compound_data_value.tps +++ b/source/expectations/data_values/ut_compound_data_value.tps @@ -1,4 +1,4 @@ -create or replace type ut_compound_data_value under ut_data_value( +create or replace type ut_compound_data_value force under ut_data_value( /* utPLSQL - Version 3 Copyright 2016 - 2017 utPLSQL Project @@ -42,8 +42,10 @@ create or replace type ut_compound_data_value under ut_data_value( overriding member function to_string return varchar2, overriding member function is_multi_line return boolean, overriding member function compare_implementation(a_other ut_data_value) return integer, - overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2 ) return varchar2, + overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean := false ) return varchar2, + member function get_data_diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean ) return clob, member function get_data_diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2 ) return clob, - member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2) return integer + member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2) return integer, + member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean ) return integer ) not final not instantiable / diff --git a/source/expectations/data_values/ut_data_value.tpb b/source/expectations/data_values/ut_data_value.tpb index 605f492f3..8bc3a87aa 100644 --- a/source/expectations/data_values/ut_data_value.tpb +++ b/source/expectations/data_values/ut_data_value.tpb @@ -24,8 +24,8 @@ create or replace type body ut_data_value as begin return false; end; - - member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2 ) return varchar2 is + + member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean :=false ) return varchar2 is begin return null; end; diff --git a/source/expectations/data_values/ut_data_value.tps b/source/expectations/data_values/ut_data_value.tps index fba0ed3c4..c9ae28d5e 100644 --- a/source/expectations/data_values/ut_data_value.tps +++ b/source/expectations/data_values/ut_data_value.tps @@ -1,4 +1,4 @@ -create or replace type ut_data_value authid current_user as object ( +create or replace type ut_data_value force authid current_user as object ( /* utPLSQL - Version 3 Copyright 2016 - 2017 utPLSQL Project @@ -24,7 +24,7 @@ create or replace type ut_data_value authid current_user as object ( final member function to_string_report(a_add_new_line_for_multi_line boolean := false, a_with_object_info boolean := true) return varchar2, order member function compare( a_other ut_data_value ) return integer, member function is_diffable return boolean, - member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2 ) return varchar2, + member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean := false ) return varchar2, not instantiable member function compare_implementation( a_other ut_data_value ) return integer ) not final not instantiable / diff --git a/source/expectations/data_values/ut_data_value_anydata.tps b/source/expectations/data_values/ut_data_value_anydata.tps index 91d66c7bd..87829cfa1 100644 --- a/source/expectations/data_values/ut_data_value_anydata.tps +++ b/source/expectations/data_values/ut_data_value_anydata.tps @@ -1,4 +1,4 @@ -create or replace type ut_data_value_anydata under ut_compound_data_value( +create or replace type ut_data_value_anydata force under ut_compound_data_value( /* utPLSQL - Version 3 Copyright 2016 - 2017 utPLSQL Project diff --git a/source/expectations/data_values/ut_data_value_refcursor.tpb b/source/expectations/data_values/ut_data_value_refcursor.tpb index ccebdb99e..5df2c76eb 100644 --- a/source/expectations/data_values/ut_data_value_refcursor.tpb +++ b/source/expectations/data_values/ut_data_value_refcursor.tpb @@ -108,7 +108,7 @@ create or replace type body ut_data_value_refcursor as return l_result_string; end; - overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2 ) return varchar2 is + overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean := false ) return varchar2 is l_result clob; l_results ut_utils.t_clob_tab := ut_utils.t_clob_tab(); l_result_string varchar2(32767); @@ -176,8 +176,12 @@ create or replace type body ut_data_value_refcursor as end if; --diff rows and row elements - ut_utils.append_to_clob(l_result, self.get_data_diff(a_other, l_exclude_xpath, a_include_xpath)); - + if a_unordered then + ut_utils.append_to_clob(l_result, self.get_data_diff(a_other, l_exclude_xpath, a_include_xpath, a_unordered)); + else + ut_utils.append_to_clob(l_result, self.get_data_diff(a_other, l_exclude_xpath, a_include_xpath)); + end if; + l_result_string := ut_utils.to_string(l_result,null); dbms_lob.freetemporary(l_result); return l_result_string; @@ -203,5 +207,26 @@ create or replace type body ut_data_value_refcursor as return l_result; end; + overriding member function compare_implementation (a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean) return integer is + l_result integer := 0; + l_other ut_data_value_refcursor; + begin + if not a_other is of (ut_data_value_refcursor) then + raise value_error; + end if; + + l_other := treat(a_other as ut_data_value_refcursor); + + --if column names/types are not equal - build a diff of column names and types + if ut_compound_data_helper.columns_hash( self, a_exclude_xpath, a_include_xpath ) + != ut_compound_data_helper.columns_hash( l_other, a_exclude_xpath, a_include_xpath ) + then + l_result := 1; + end if; + l_result := l_result + (self as ut_compound_data_value).compare_implementation(a_other, a_exclude_xpath, a_include_xpath, a_unordered); + return l_result; + end; + + end; / diff --git a/source/expectations/data_values/ut_data_value_refcursor.tps b/source/expectations/data_values/ut_data_value_refcursor.tps index c59bf2662..679030fe5 100644 --- a/source/expectations/data_values/ut_data_value_refcursor.tps +++ b/source/expectations/data_values/ut_data_value_refcursor.tps @@ -33,7 +33,9 @@ create or replace type ut_data_value_refcursor under ut_compound_data_value( constructor function ut_data_value_refcursor(self in out nocopy ut_data_value_refcursor, a_value sys_refcursor) return self as result, member procedure init(self in out nocopy ut_data_value_refcursor, a_value sys_refcursor), overriding member function to_string return varchar2, - overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2 ) return varchar2, - overriding member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2) return integer + overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean := false ) return varchar2, + overriding member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2) return integer, + overriding member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean) return integer + ) / diff --git a/source/expectations/matchers/ut_equal.tpb b/source/expectations/matchers/ut_equal.tpb index 29b8f511d..09b1b7e5e 100644 --- a/source/expectations/matchers/ut_equal.tpb +++ b/source/expectations/matchers/ut_equal.tpb @@ -182,6 +182,13 @@ create or replace type body ut_equal as return l_result; end; + member function unordered return ut_equal is + l_result ut_equal := self; + begin + l_result.is_unordered := 'true'; + return l_result; + end; + member function get_include_xpath return varchar2 is begin return ut_utils.to_xpath( coalesce(include_list, ut_varchar2_list()) ); @@ -192,6 +199,12 @@ create or replace type body ut_equal as return ut_utils.to_xpath( coalesce(exclude_list, ut_varchar2_list()) ); end; + member function get_unordered return boolean is + begin + return case when is_unordered = 'true' then true else false end; + end; + + overriding member function run_matcher(self in out nocopy ut_equal, a_actual ut_data_value) return boolean is l_result boolean; begin @@ -199,7 +212,11 @@ create or replace type body ut_equal as if self.expected is of (ut_data_value_anydata) then l_result := 0 = treat(self.expected as ut_data_value_anydata).compare_implementation(a_actual, get_exclude_xpath(), get_include_xpath()); elsif self.expected is of (ut_data_value_refcursor) then - l_result := 0 = treat(self.expected as ut_data_value_refcursor).compare_implementation(a_actual, get_exclude_xpath(), get_include_xpath()); + if get_unordered then + l_result := 0 = treat(self.expected as ut_data_value_refcursor).compare_implementation(a_actual, get_exclude_xpath(), get_include_xpath(),get_unordered()); + else + l_result := 0 = treat(self.expected as ut_data_value_refcursor).compare_implementation(a_actual, get_exclude_xpath(), get_include_xpath()); + end if; else l_result := equal_with_nulls((self.expected = a_actual), a_actual); end if; @@ -216,7 +233,7 @@ create or replace type body ut_equal as if self.expected.data_type = a_actual.data_type and self.expected.is_diffable then l_result := 'Actual: '||a_actual.get_object_info()||' '||self.description()||': '||self.expected.get_object_info() - || chr(10) || 'Diff:' || expected.diff(a_actual, get_exclude_xpath(), get_include_xpath()); + || chr(10) || 'Diff:' || expected.diff(a_actual, get_exclude_xpath(), get_include_xpath(), get_unordered()); else l_result := (self as ut_matcher).failure_message(a_actual) || ': '|| self.expected.to_string_report(); end if; diff --git a/source/expectations/matchers/ut_equal.tps b/source/expectations/matchers/ut_equal.tps index 710b23f7f..a17dceb1d 100644 --- a/source/expectations/matchers/ut_equal.tps +++ b/source/expectations/matchers/ut_equal.tps @@ -25,7 +25,10 @@ create or replace type ut_equal under ut_comparison_matcher( * Holds (list of columns/attributes) to incude when comparing compound types */ include_list ut_varchar2_list, + + is_unordered varchar2(5), + member procedure init(self in out nocopy ut_equal, a_expected ut_data_value, a_nulls_are_equal boolean), member function equal_with_nulls( self in ut_equal, a_assert_result boolean, a_actual ut_data_value) return boolean, constructor function ut_equal(self in out nocopy ut_equal, a_expected anydata, a_nulls_are_equal boolean := null) return self as result, @@ -49,8 +52,10 @@ create or replace type ut_equal under ut_comparison_matcher( member function include(a_items ut_varchar2_list) return ut_equal, member function exclude(a_items varchar2) return ut_equal, member function exclude(a_items ut_varchar2_list) return ut_equal, + member function unordered return ut_equal, member function get_include_xpath return varchar2, member function get_exclude_xpath return varchar2, + member function get_unordered return boolean, overriding member function run_matcher(self in out nocopy ut_equal, a_actual ut_data_value) return boolean, overriding member function failure_message(a_actual ut_data_value) return varchar2, overriding member function failure_message_when_negated(a_actual ut_data_value) return varchar2 diff --git a/source/expectations/ut_expectation_compound.tpb b/source/expectations/ut_expectation_compound.tpb index 8665af265..d08b74186 100644 --- a/source/expectations/ut_expectation_compound.tpb +++ b/source/expectations/ut_expectation_compound.tpb @@ -146,6 +146,24 @@ create or replace type body ut_expectation_compound as self.to_( treat(matcher as ut_equal).exclude(a_items) ); end if; end; + + member function unordered return ut_expectation_compound is + l_result ut_expectation_compound; + begin + l_result := self; + l_result.matcher := treat(l_result.matcher as ut_equal).unordered; + return l_result; + end; + + member procedure unordered(self in ut_expectation_compound) is + begin + + if ut_utils.int_to_boolean(negated) then + self.not_to( treat(matcher as ut_equal).unordered ); + else + self.to_( treat(matcher as ut_equal).unordered ); + end if; + end; end; / diff --git a/source/expectations/ut_expectation_compound.tps b/source/expectations/ut_expectation_compound.tps index 66dcb91af..27a0fd22d 100644 --- a/source/expectations/ut_expectation_compound.tps +++ b/source/expectations/ut_expectation_compound.tps @@ -36,8 +36,9 @@ create or replace type ut_expectation_compound under ut_expectation( member function exclude(a_items varchar2) return ut_expectation_compound, member function exclude(a_items ut_varchar2_list) return ut_expectation_compound, member procedure exclude(self in ut_expectation_compound, a_items varchar2), - member procedure exclude(self in ut_expectation_compound, a_items ut_varchar2_list) - + member procedure exclude(self in ut_expectation_compound, a_items ut_varchar2_list), + member function unordered return ut_expectation_compound, + member procedure unordered(self in ut_expectation_compound) ) final / From 41ae0ffcae54f32af259ec712a861c701f1bbf05 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Tue, 24 Apr 2018 22:21:25 +0100 Subject: [PATCH 02/24] Added methods for join_by Adding compare for join by --- .../data_values/ut_compound_data_diff_tmp.sql | 8 +- .../data_values/ut_compound_data_helper.pkb | 126 +++++++++++++++--- .../data_values/ut_compound_data_helper.pks | 6 + .../data_values/ut_compound_data_tmp.sql | 5 +- .../data_values/ut_compound_data_value.tpb | 68 +++++++--- .../data_values/ut_compound_data_value.tps | 6 +- .../data_values/ut_data_value.tpb | 2 +- .../data_values/ut_data_value.tps | 2 +- .../data_values/ut_data_value_refcursor.tpb | 20 +-- .../data_values/ut_data_value_refcursor.tps | 6 +- source/expectations/matchers/ut_equal.tpb | 29 +++- source/expectations/matchers/ut_equal.tps | 12 +- .../expectations/ut_expectation_compound.tpb | 35 +++++ .../expectations/ut_expectation_compound.tps | 10 +- 14 files changed, 271 insertions(+), 64 deletions(-) diff --git a/source/expectations/data_values/ut_compound_data_diff_tmp.sql b/source/expectations/data_values/ut_compound_data_diff_tmp.sql index 881cc8082..c10226f2c 100644 --- a/source/expectations/data_values/ut_compound_data_diff_tmp.sql +++ b/source/expectations/data_values/ut_compound_data_diff_tmp.sql @@ -14,5 +14,11 @@ create global temporary table ut_compound_data_diff_tmp( */ diff_id raw(128), item_no integer, - constraint ut_compound_data_diff_tmp_pk primary key(diff_id,item_no) + item_hash raw(128), + duplicate_no integer, + --constraint ut_compound_data_diff_tmp_uk1 unique (diff_id, item_no, item_hash, duplicate_no), + constraint ut_compound_data_diff_tmp_chk check( + item_no is not null and item_hash is null and duplicate_no is null + or item_no is null and item_hash is not null and duplicate_no is not null + ) ) on commit preserve rows; diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index ed9e397d2..d66c7c80f 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -160,6 +160,79 @@ create or replace package body ut_compound_data_helper is return l_results; end; + function get_rows_diff( + a_expected_dataset_guid raw, a_actual_dataset_guid raw, a_diff_id raw, + a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2, + a_join_by_xpath varchar2 + ) return tt_row_diffs is + l_column_filter varchar2(32767); + l_results tt_row_diffs; + l_sql varchar2(32767); -- REMOVE LATER also for unorder + begin + l_column_filter := get_columns_filter(a_exclude_xpath,a_include_xpath); + + /** + * Since its unordered search we cannot select max rows from diffs as we miss some comparision records + * We will restrict output on higher level of select + */ + + l_sql := q'[ + with + diff_info as (select item_hash from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) + select rn,diff_type,diffed_row + from (select dense_rank() over (order by pk_hash) as rn, diff_type,data_item diffed_row + from (select nvl(exp.pk_hash, act.pk_hash) pk_hash, + xmlserialize(content exp.row_data no indent) exp_item, + xmlserialize(content act.row_data no indent) act_item + from + (select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no + from + (select ucd.column_value row_data, + dbms_crypto.hash( value(ucd).getclobval(),3) row_hash, + dbms_crypto.hash( extract(value(ucd),']'|| a_join_by_xpath ||q'[').getClobVal(),3/*HASH_SH1*/) pk_hash + from + (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter + from ut_compound_data_tmp ucd + where ucd.data_id = :self_guid + and ucd.item_hash in (select i.item_hash from diff_info i) + ) r, + table( xmlsequence( extract(r.item_data,'/*') ) ) ucd + ) ucd + ) exp + join ( + select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no + from + (select ucd.column_value row_data, + dbms_crypto.hash( value(ucd).getclobval(),3/*HASH_SH1*/) row_hash, + dbms_crypto.hash( extract(value(ucd),']'|| a_join_by_xpath ||q'[').getClobVal(),3/*HASH_SH1*/) pk_hash + from + (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter + from ut_compound_data_tmp ucd + where ucd.data_id = :other_guid + and ucd.item_hash in (select i.item_hash from diff_info i) + ) r, + table( xmlsequence( extract(r.item_data,'/*') ) ) ucd + ) ucd + ) act + on exp.pk_hash = act.pk_hash and exp.duplicate_no = act.duplicate_no + where exp.row_hash != act.row_hash + or exp.row_hash is null + or act.row_hash is null + ) + unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:') ) + ) + where rownum < :max_rows + order by 1, 2]'; + + execute immediate l_sql + bulk collect into l_results + using a_diff_id, + a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, + a_exclude_xpath, a_include_xpath, a_actual_dataset_guid, + a_max_rows; + return l_results; + end; + function get_rows_diff( a_expected_dataset_guid raw, a_actual_dataset_guid raw, a_diff_id raw, a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2 @@ -237,21 +310,36 @@ create or replace package body ut_compound_data_helper is l_results tt_row_diffs; begin l_column_filter := get_columns_filter(a_exclude_xpath,a_include_xpath); - execute immediate q'[ - select - coalesce(exp.duplicate_no, act.duplicate_no) duplicate_no, - case when exp.row_hash is null then 'Actual:' else 'Expected:' end diffed_type, - case when exp.row_hash is null then - xmlserialize(content act.row_data no indent) - else - xmlserialize(content exp.row_data no indent) - end diffed_row - from (select ucd.*, row_number() over(partition by row_hash order by row_hash) duplicate_no + + /** + * Since its unordered search we cannot select max rows from diffs as we miss some comparision records + * We will restrict output on higher level of select + */ + execute immediate q'[with + diff_info as (select item_hash from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) + select duplicate_no, + diffed_type, + diffed_row + from + (select + coalesce(exp.duplicate_no,act.duplicate_no) duplicate_no, + case + when act.row_hash is null then + 'miss row' + else 'extra row' + end diffed_type, + case when exp.row_hash is null then + xmlserialize(content act.row_data no indent) + when act.row_hash is null then + xmlserialize(content exp.row_data no indent) + end diffed_row + from (select ucd.*, row_number() over(partition by row_hash order by row_hash) duplicate_no from (select ucd.column_value row_data, dbms_crypto.hash( value(ucd).getclobval(),3) row_hash - from (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter + from (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter from ut_compound_data_tmp ucd where ucd.data_id = :self_guid + and ucd.item_hash in (select i.item_hash from diff_info i) ) r, table( xmlsequence( extract(r.item_data,'/*') ) ) ucd ) ucd @@ -260,21 +348,23 @@ create or replace package body ut_compound_data_helper is (select ucd.*, row_number() over(partition by row_hash order by row_hash) duplicate_no from (select ucd.column_value row_data, dbms_crypto.hash( value(ucd).getclobval(),3/*HASH_SH1*/) row_hash - from (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter + from (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter from ut_compound_data_tmp ucd where ucd.data_id = :other_guid + and ucd.item_hash in (select i.item_hash from diff_info i) ) r, table( xmlsequence( extract(r.item_data,'/*') ) ) ucd ) ucd ) act - on exp.row_hash = act.row_hash - and exp.duplicate_no = act.duplicate_no - where exp.row_hash is null or act.row_hash is null]' + on exp.row_hash = act.row_hash + and exp.duplicate_no = act.duplicate_no + where exp.row_hash is null or act.row_hash is null ) where rownum < :max_rows ]' bulk collect into l_results - using a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, - a_exclude_xpath, a_include_xpath, a_actual_dataset_guid; + using a_diff_id, + a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, + a_exclude_xpath, a_include_xpath, a_actual_dataset_guid, + a_max_rows; - --execute immediate 'create table test as select * from ut_compound_data_tmp'; return l_results; end; diff --git a/source/expectations/data_values/ut_compound_data_helper.pks b/source/expectations/data_values/ut_compound_data_helper.pks index ae9b2a044..3f8595233 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pks +++ b/source/expectations/data_values/ut_compound_data_helper.pks @@ -52,6 +52,12 @@ create or replace package ut_compound_data_helper authid definer is a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2 ) return tt_row_diffs; + function get_rows_diff( + a_expected_dataset_guid raw, a_actual_dataset_guid raw, a_diff_id raw, + a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2, + a_join_by_xpath varchar2 + ) return tt_row_diffs; + function get_rows_diff_unordered( a_expected_dataset_guid raw, a_actual_dataset_guid raw, a_diff_id raw, a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2 diff --git a/source/expectations/data_values/ut_compound_data_tmp.sql b/source/expectations/data_values/ut_compound_data_tmp.sql index 300b0fd38..8af8083ef 100644 --- a/source/expectations/data_values/ut_compound_data_tmp.sql +++ b/source/expectations/data_values/ut_compound_data_tmp.sql @@ -15,5 +15,8 @@ create global temporary table ut_compound_data_tmp( data_id raw(32), item_no integer, item_data xmltype, - constraint ut_compound_data_tmp_pk primary key(data_id, item_no) + item_hash raw(128), + duplicate_no integer + --constraint ut_cmp_data_tmp_item_ak unique (data_id, item_no), + --constraint ut_cmp_data_tmp_hash_pk unique (data_id, item_hash , duplicate_no) ) on commit preserve rows; diff --git a/source/expectations/data_values/ut_compound_data_value.tpb b/source/expectations/data_values/ut_compound_data_value.tpb index 8a5fb2ab3..cd3af7ac4 100644 --- a/source/expectations/data_values/ut_compound_data_value.tpb +++ b/source/expectations/data_values/ut_compound_data_value.tpb @@ -71,11 +71,17 @@ create or replace type body ut_compound_data_value as return l_result_string; end; - overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean := false ) return varchar2 is + overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean := false ) return varchar2 is l_result clob; l_result_string varchar2(32767); begin - l_result := get_data_diff(a_other, a_exclude_xpath, a_include_xpath, a_unordered); + if a_join_by_xpath is not null then + l_result := get_data_diff(a_other, a_exclude_xpath, a_include_xpath, a_join_by_xpath); + elsif a_unordered then + l_result := get_data_diff(a_other, a_exclude_xpath, a_include_xpath, a_unordered); + else + l_result := get_data_diff(a_other, a_exclude_xpath, a_include_xpath, a_join_by_xpath); + end if; l_result_string := ut_utils.to_string(l_result,null); dbms_lob.freetemporary(l_result); return l_result_string; @@ -102,8 +108,8 @@ create or replace type body ut_compound_data_value as --diff rows and row elements l_diff_id := ut_compound_data_helper.get_hash(self.data_id||l_actual.data_id); -- First tell how many rows are different - execute immediate 'select unordered_cnt from ' || l_ut_owner || '.ut_compound_data_diff_tmp where diff_id = :diff_id' into l_diff_row_count using l_diff_id; - + execute immediate 'select count(*) from ' || l_ut_owner || '.ut_compound_data_diff_tmp where diff_id = :diff_id' into l_diff_row_count using l_diff_id; + if l_diff_row_count > 0 then l_row_diffs := ut_compound_data_helper.get_rows_diff_unordered( self.data_id, l_actual.data_id, l_diff_id, c_max_rows, a_exclude_xpath, a_include_xpath @@ -118,14 +124,14 @@ create or replace type body ut_compound_data_value as for i in 1 .. l_row_diffs.count loop l_results.extend; l_results(l_results.last) := - ' Row Appeared '||l_row_diffs(i).rn||' - '||rpad(l_row_diffs(i).diff_type,10)||l_row_diffs(i).diffed_row; + rpad(l_row_diffs(i).diff_type,10)||l_row_diffs(i).diffed_row; end loop; ut_utils.append_to_clob(l_result,l_results); end if; return l_result; end; - member function get_data_diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2 ) return clob is + member function get_data_diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2) return clob is c_max_rows constant integer := 20; l_result clob; l_results ut_utils.t_clob_tab := ut_utils.t_clob_tab(); @@ -149,9 +155,15 @@ create or replace type body ut_compound_data_value as execute immediate 'select count(*) from ' || l_ut_owner || '.ut_compound_data_diff_tmp where diff_id = :diff_id' into l_diff_row_count using l_diff_id; if l_diff_row_count > 0 then - l_row_diffs := ut_compound_data_helper.get_rows_diff( + if a_join_by_xpath is not null then + l_row_diffs := ut_compound_data_helper.get_rows_diff( + self.data_id, l_actual.data_id, l_diff_id, c_max_rows, a_exclude_xpath, a_include_xpath, a_join_by_xpath + ); + else + l_row_diffs := ut_compound_data_helper.get_rows_diff( self.data_id, l_actual.data_id, l_diff_id, c_max_rows, a_exclude_xpath, a_include_xpath - ); + ); + end if; l_message := chr(10) ||'Rows: [ ' || l_diff_row_count ||' differences' || case when l_diff_row_count > c_max_rows and l_row_diffs.count > 0 then ', showing first '||c_max_rows end @@ -224,7 +236,7 @@ create or replace type body ut_compound_data_value as return l_result; end; -member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean ) return integer is + member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean ) return integer is l_other ut_compound_data_value; l_ut_owner varchar2(250) := ut_utils.ut_owner; l_column_filter varchar2(32767); @@ -241,16 +253,36 @@ member function compare_implementation(a_other ut_data_value, a_exclude_xpath va l_diff_id := ut_compound_data_helper.get_hash(self.data_id||l_other.data_id); l_column_filter := ut_compound_data_helper.get_columns_filter(a_exclude_xpath, a_include_xpath); - - -- Find differences - l_row_diffs := ut_compound_data_helper.get_rows_diff_unordered( - self.data_id, l_other.data_id, l_diff_id, c_max_rows, a_exclude_xpath, a_include_xpath - ); - - execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id, unordered_cnt ) - values (:diff_id, nvl(:unordered_cnt,0))' using l_diff_id, l_row_diffs.count; + + -- Pre generate hash minus to leave only onese that are diffrent, for example duplicates or diffrent hash + execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,duplicate_no ) + select :diff_id,x.item_hash,tmp.duplicate_no + from( + ( + select item_hash,row_number() over (partition by item_hash,data_id order by 1) duplicate_no + from ut_compound_data_tmp + where data_id = :self_guid + minus + select item_hash,row_number() over (partition by item_hash,data_id order by 1) + from ut_compound_data_tmp + where data_id = :other_guid + ) + union all + ( + select item_hash,row_number() over (partition by item_hash,data_id order by 1) + from ut_compound_data_tmp + where data_id = :other_guid + minus + select item_hash,row_number() over (partition by item_hash,data_id order by 1) + from ut_compound_data_tmp + where data_id = :self_guid + ))tmp + ,ut_compound_data_tmp x + where tmp.item_hash = x.item_hash' + using l_diff_id, self.data_id, l_other.data_id + ,l_other.data_id,self.data_id; --result is OK only if both are same - if l_row_diffs.count = 0 and self.elements_count = l_other.elements_count then + if sql%rowcount = 0 and self.elements_count = l_other.elements_count then l_result := 0; else l_result := 1; diff --git a/source/expectations/data_values/ut_compound_data_value.tps b/source/expectations/data_values/ut_compound_data_value.tps index d2345c01e..cc192bbc3 100644 --- a/source/expectations/data_values/ut_compound_data_value.tps +++ b/source/expectations/data_values/ut_compound_data_value.tps @@ -42,10 +42,10 @@ create or replace type ut_compound_data_value force under ut_data_value( overriding member function to_string return varchar2, overriding member function is_multi_line return boolean, overriding member function compare_implementation(a_other ut_data_value) return integer, - overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean := false ) return varchar2, + overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean := false ) return varchar2, member function get_data_diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean ) return clob, - member function get_data_diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2 ) return clob, + member function get_data_diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2 ) return clob, member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2) return integer, - member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean ) return integer + member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean ) return integer ) not final not instantiable / diff --git a/source/expectations/data_values/ut_data_value.tpb b/source/expectations/data_values/ut_data_value.tpb index 8bc3a87aa..d5ec11092 100644 --- a/source/expectations/data_values/ut_data_value.tpb +++ b/source/expectations/data_values/ut_data_value.tpb @@ -25,7 +25,7 @@ create or replace type body ut_data_value as return false; end; - member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean :=false ) return varchar2 is + member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean :=false ) return varchar2 is begin return null; end; diff --git a/source/expectations/data_values/ut_data_value.tps b/source/expectations/data_values/ut_data_value.tps index c9ae28d5e..79ac4eef9 100644 --- a/source/expectations/data_values/ut_data_value.tps +++ b/source/expectations/data_values/ut_data_value.tps @@ -24,7 +24,7 @@ create or replace type ut_data_value force authid current_user as object ( final member function to_string_report(a_add_new_line_for_multi_line boolean := false, a_with_object_info boolean := true) return varchar2, order member function compare( a_other ut_data_value ) return integer, member function is_diffable return boolean, - member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean := false ) return varchar2, + member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean := false ) return varchar2, not instantiable member function compare_implementation( a_other ut_data_value ) return integer ) not final not instantiable / diff --git a/source/expectations/data_values/ut_data_value_refcursor.tpb b/source/expectations/data_values/ut_data_value_refcursor.tpb index 5df2c76eb..cabe18011 100644 --- a/source/expectations/data_values/ut_data_value_refcursor.tpb +++ b/source/expectations/data_values/ut_data_value_refcursor.tpb @@ -60,8 +60,8 @@ create or replace type body ut_data_value_refcursor as l_xml := dbms_xmlgen.getxmltype(l_ctx); execute immediate - 'insert into ' || l_ut_owner || '.ut_compound_data_tmp(data_id, item_no, item_data) ' || - 'select :self_guid, :self_row_count + rownum, value(a) ' || + 'insert into ' || l_ut_owner || '.ut_compound_data_tmp(data_id, item_no, item_data, item_hash) ' || + 'select :self_guid, :self_row_count + rownum, value(a), dbms_crypto.hash( value(a).getclobval(),3)' || ' from table( xmlsequence( extract(:l_xml,''ROWSET/*'') ) ) a' using in self.data_id, self.elements_count, l_xml; @@ -69,7 +69,7 @@ create or replace type body ut_data_value_refcursor as self.elements_count := self.elements_count + sql%rowcount; end loop; - + ut_expectation_processor.reset_nls_params(); if l_cursor%isopen then close l_cursor; @@ -108,7 +108,7 @@ create or replace type body ut_data_value_refcursor as return l_result_string; end; - overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean := false ) return varchar2 is + overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean := false ) return varchar2 is l_result clob; l_results ut_utils.t_clob_tab := ut_utils.t_clob_tab(); l_result_string varchar2(32767); @@ -176,10 +176,12 @@ create or replace type body ut_data_value_refcursor as end if; --diff rows and row elements - if a_unordered then + if a_join_by_xpath is not null then + ut_utils.append_to_clob(l_result, self.get_data_diff(a_other, a_exclude_xpath, a_include_xpath, a_join_by_xpath)); + elsif a_unordered then ut_utils.append_to_clob(l_result, self.get_data_diff(a_other, l_exclude_xpath, a_include_xpath, a_unordered)); else - ut_utils.append_to_clob(l_result, self.get_data_diff(a_other, l_exclude_xpath, a_include_xpath)); + ut_utils.append_to_clob(l_result, self.get_data_diff(a_other, l_exclude_xpath, a_include_xpath, a_join_by_xpath)); end if; l_result_string := ut_utils.to_string(l_result,null); @@ -196,7 +198,6 @@ create or replace type body ut_data_value_refcursor as end if; l_other := treat(a_other as ut_data_value_refcursor); - --if column names/types are not equal - build a diff of column names and types if ut_compound_data_helper.columns_hash( self, a_exclude_xpath, a_include_xpath ) != ut_compound_data_helper.columns_hash( l_other, a_exclude_xpath, a_include_xpath ) @@ -207,7 +208,7 @@ create or replace type body ut_data_value_refcursor as return l_result; end; - overriding member function compare_implementation (a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean) return integer is + overriding member function compare_implementation (a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean) return integer is l_result integer := 0; l_other ut_data_value_refcursor; begin @@ -216,14 +217,13 @@ create or replace type body ut_data_value_refcursor as end if; l_other := treat(a_other as ut_data_value_refcursor); - --if column names/types are not equal - build a diff of column names and types if ut_compound_data_helper.columns_hash( self, a_exclude_xpath, a_include_xpath ) != ut_compound_data_helper.columns_hash( l_other, a_exclude_xpath, a_include_xpath ) then l_result := 1; end if; - l_result := l_result + (self as ut_compound_data_value).compare_implementation(a_other, a_exclude_xpath, a_include_xpath, a_unordered); + l_result := l_result + (self as ut_compound_data_value).compare_implementation(a_other, a_exclude_xpath, a_include_xpath, a_join_by_xpath, a_unordered); return l_result; end; diff --git a/source/expectations/data_values/ut_data_value_refcursor.tps b/source/expectations/data_values/ut_data_value_refcursor.tps index 679030fe5..af0609bb6 100644 --- a/source/expectations/data_values/ut_data_value_refcursor.tps +++ b/source/expectations/data_values/ut_data_value_refcursor.tps @@ -33,9 +33,9 @@ create or replace type ut_data_value_refcursor under ut_compound_data_value( constructor function ut_data_value_refcursor(self in out nocopy ut_data_value_refcursor, a_value sys_refcursor) return self as result, member procedure init(self in out nocopy ut_data_value_refcursor, a_value sys_refcursor), overriding member function to_string return varchar2, - overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean := false ) return varchar2, + overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean := false ) return varchar2, overriding member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2) return integer, - overriding member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_unordered boolean) return integer + overriding member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean) return integer ) -/ +/ \ No newline at end of file diff --git a/source/expectations/matchers/ut_equal.tpb b/source/expectations/matchers/ut_equal.tpb index 09b1b7e5e..f9e6baaeb 100644 --- a/source/expectations/matchers/ut_equal.tpb +++ b/source/expectations/matchers/ut_equal.tpb @@ -23,6 +23,7 @@ create or replace type body ut_equal as self.expected := a_expected; self.include_list := ut_varchar2_list(); self.exclude_list := ut_varchar2_list(); + self.join_columns := ut_varchar2_list(); end; member function equal_with_nulls(a_assert_result boolean, a_actual ut_data_value) return boolean is @@ -185,7 +186,23 @@ create or replace type body ut_equal as member function unordered return ut_equal is l_result ut_equal := self; begin - l_result.is_unordered := 'true'; + l_result.is_unordered := ut_utils.boolean_to_int(true); + return l_result; + end; + + member function join_by(a_columns varchar2) return ut_equal is + l_result ut_equal := self; + begin + l_result.is_unordered := ut_utils.boolean_to_int(true); + ut_utils.append_to_list(l_result.join_columns, a_columns); + return l_result; + end; + + member function join_by(a_columns ut_varchar2_list) return ut_equal is + l_result ut_equal := self; + begin + l_result.is_unordered := ut_utils.boolean_to_int(true); + l_result.join_columns := l_result.join_columns multiset union all coalesce(a_columns,ut_varchar2_list()); return l_result; end; @@ -201,9 +218,13 @@ create or replace type body ut_equal as member function get_unordered return boolean is begin - return case when is_unordered = 'true' then true else false end; + return ut_utils.int_to_boolean(nvl(is_unordered,0)); end; + member function get_join_by_xpath return varchar2 is + begin + return ut_utils.to_xpath( coalesce(join_columns, ut_varchar2_list()) ); + end; overriding member function run_matcher(self in out nocopy ut_equal, a_actual ut_data_value) return boolean is l_result boolean; @@ -213,7 +234,7 @@ create or replace type body ut_equal as l_result := 0 = treat(self.expected as ut_data_value_anydata).compare_implementation(a_actual, get_exclude_xpath(), get_include_xpath()); elsif self.expected is of (ut_data_value_refcursor) then if get_unordered then - l_result := 0 = treat(self.expected as ut_data_value_refcursor).compare_implementation(a_actual, get_exclude_xpath(), get_include_xpath(),get_unordered()); + l_result := 0 = treat(self.expected as ut_data_value_refcursor).compare_implementation(a_actual, get_exclude_xpath(), get_include_xpath(), get_join_by_xpath(), get_unordered()); else l_result := 0 = treat(self.expected as ut_data_value_refcursor).compare_implementation(a_actual, get_exclude_xpath(), get_include_xpath()); end if; @@ -233,7 +254,7 @@ create or replace type body ut_equal as if self.expected.data_type = a_actual.data_type and self.expected.is_diffable then l_result := 'Actual: '||a_actual.get_object_info()||' '||self.description()||': '||self.expected.get_object_info() - || chr(10) || 'Diff:' || expected.diff(a_actual, get_exclude_xpath(), get_include_xpath(), get_unordered()); + || chr(10) || 'Diff:' || expected.diff(a_actual, get_exclude_xpath(), get_include_xpath(), get_join_by_xpath(), get_unordered()); else l_result := (self as ut_matcher).failure_message(a_actual) || ': '|| self.expected.to_string_report(); end if; diff --git a/source/expectations/matchers/ut_equal.tps b/source/expectations/matchers/ut_equal.tps index a17dceb1d..01b1d2e1e 100644 --- a/source/expectations/matchers/ut_equal.tps +++ b/source/expectations/matchers/ut_equal.tps @@ -26,8 +26,15 @@ create or replace type ut_equal under ut_comparison_matcher( */ include_list ut_varchar2_list, - is_unordered varchar2(5), + /** + * Holds value if comparision on refcursor to be performed as unordered set + */ + is_unordered number(1,0), + /** + * Holds list of columns to be used as a join PK on sys_refcursor comparision + */ + join_columns ut_varchar2_list, member procedure init(self in out nocopy ut_equal, a_expected ut_data_value, a_nulls_are_equal boolean), member function equal_with_nulls( self in ut_equal, a_assert_result boolean, a_actual ut_data_value) return boolean, @@ -53,9 +60,12 @@ create or replace type ut_equal under ut_comparison_matcher( member function exclude(a_items varchar2) return ut_equal, member function exclude(a_items ut_varchar2_list) return ut_equal, member function unordered return ut_equal, + member function join_by(a_columns varchar2) return ut_equal, + member function join_by(a_columns ut_varchar2_list) return ut_equal, member function get_include_xpath return varchar2, member function get_exclude_xpath return varchar2, member function get_unordered return boolean, + member function get_join_by_xpath return varchar2, overriding member function run_matcher(self in out nocopy ut_equal, a_actual ut_data_value) return boolean, overriding member function failure_message(a_actual ut_data_value) return varchar2, overriding member function failure_message_when_negated(a_actual ut_data_value) return varchar2 diff --git a/source/expectations/ut_expectation_compound.tpb b/source/expectations/ut_expectation_compound.tpb index d08b74186..dea146bd2 100644 --- a/source/expectations/ut_expectation_compound.tpb +++ b/source/expectations/ut_expectation_compound.tpb @@ -165,5 +165,40 @@ create or replace type body ut_expectation_compound as end if; end; + member function join_by(a_columns varchar2) return ut_expectation_compound is + l_result ut_expectation_compound; + begin + l_result := self; + l_result.matcher := treat(l_result.matcher as ut_equal).join_by(a_columns); + return l_result; + end; + + member function join_by(a_columns ut_varchar2_list) return ut_expectation_compound is + l_result ut_expectation_compound; + begin + l_result := self; + l_result.matcher := treat(l_result.matcher as ut_equal).join_by(a_columns); + return l_result; + end; + + member procedure join_by(self in ut_expectation_compound, a_columns varchar2) is + begin + if ut_utils.int_to_boolean(negated) then + self.not_to( treat(matcher as ut_equal).join_by(a_columns) ); + else + self.to_( treat(matcher as ut_equal).join_by(a_columns) ); + end if; + end; + + member procedure join_by(self in ut_expectation_compound, a_columns ut_varchar2_list) is + begin + + if ut_utils.int_to_boolean(negated) then + self.not_to( treat(matcher as ut_equal).join_by(a_columns) ); + else + self.to_( treat(matcher as ut_equal).join_by(a_columns) ); + end if; + end; + end; / diff --git a/source/expectations/ut_expectation_compound.tps b/source/expectations/ut_expectation_compound.tps index 27a0fd22d..967706a21 100644 --- a/source/expectations/ut_expectation_compound.tps +++ b/source/expectations/ut_expectation_compound.tps @@ -37,8 +37,12 @@ create or replace type ut_expectation_compound under ut_expectation( member function exclude(a_items ut_varchar2_list) return ut_expectation_compound, member procedure exclude(self in ut_expectation_compound, a_items varchar2), member procedure exclude(self in ut_expectation_compound, a_items ut_varchar2_list), - member function unordered return ut_expectation_compound, - member procedure unordered(self in ut_expectation_compound) + member function unordered return ut_expectation_compound, + member procedure unordered(self in ut_expectation_compound), + member function join_by(a_columns varchar2) return ut_expectation_compound, + member function join_by(a_columns ut_varchar2_list) return ut_expectation_compound, + member procedure join_by(self in ut_expectation_compound, a_columns varchar2), + member procedure join_by(self in ut_expectation_compound, a_columns ut_varchar2_list) ) final -/ +/ \ No newline at end of file From a3406eec8ecc7e3d4596e7f5a25a90d829796af5 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Thu, 3 May 2018 08:47:29 +0100 Subject: [PATCH 03/24] Updated JoinBy query --- .../data_values/ut_compound_data_helper.pkb | 116 ++++++++++-------- .../data_values/ut_compound_data_value.tpb | 12 +- .../data_values/ut_data_value_refcursor.tpb | 2 +- 3 files changed, 70 insertions(+), 60 deletions(-) diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index d66c7c80f..df5f5453b 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -167,68 +167,76 @@ create or replace package body ut_compound_data_helper is ) return tt_row_diffs is l_column_filter varchar2(32767); l_results tt_row_diffs; - l_sql varchar2(32767); -- REMOVE LATER also for unorder begin l_column_filter := get_columns_filter(a_exclude_xpath,a_include_xpath); /** * Since its unordered search we cannot select max rows from diffs as we miss some comparision records * We will restrict output on higher level of select - */ + **/ - l_sql := q'[ - with - diff_info as (select item_hash from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) + execute immediate q'[ + with diff_info as (select item_hash from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) select rn,diff_type,diffed_row - from (select dense_rank() over (order by pk_hash) as rn, diff_type,data_item diffed_row - from (select nvl(exp.pk_hash, act.pk_hash) pk_hash, - xmlserialize(content exp.row_data no indent) exp_item, - xmlserialize(content act.row_data no indent) act_item - from - (select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no - from - (select ucd.column_value row_data, - dbms_crypto.hash( value(ucd).getclobval(),3) row_hash, - dbms_crypto.hash( extract(value(ucd),']'|| a_join_by_xpath ||q'[').getClobVal(),3/*HASH_SH1*/) pk_hash - from - (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter - from ut_compound_data_tmp ucd - where ucd.data_id = :self_guid - and ucd.item_hash in (select i.item_hash from diff_info i) - ) r, - table( xmlsequence( extract(r.item_data,'/*') ) ) ucd - ) ucd - ) exp - join ( - select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no - from - (select ucd.column_value row_data, - dbms_crypto.hash( value(ucd).getclobval(),3/*HASH_SH1*/) row_hash, - dbms_crypto.hash( extract(value(ucd),']'|| a_join_by_xpath ||q'[').getClobVal(),3/*HASH_SH1*/) pk_hash - from - (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter - from ut_compound_data_tmp ucd - where ucd.data_id = :other_guid - and ucd.item_hash in (select i.item_hash from diff_info i) - ) r, - table( xmlsequence( extract(r.item_data,'/*') ) ) ucd - ) ucd - ) act - on exp.pk_hash = act.pk_hash and exp.duplicate_no = act.duplicate_no - where exp.row_hash != act.row_hash - or exp.row_hash is null - or act.row_hash is null - ) - unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:') ) - ) - where rownum < :max_rows - order by 1, 2]'; - - execute immediate l_sql + from + (select dense_rank() over (order by pk_hash) as rn, diff_type,data_item diffed_row + from + (select pk_hash + ,case when exp_item is not null and act_item is not null then exp_item else null end exp_item + ,case when exp_item is not null and act_item is not null then act_item else null end act_item + ,case when exp_item is not null and act_item is null then exp_item else null end miss_item + ,case when exp_item is null and act_item is not null then act_item else null end ext_item + from + (select nvl(exp.pk_hash, act.pk_hash) pk_hash, + xmlserialize(content exp.row_data no indent) exp_item, + xmlserialize(content act.row_data no indent) act_item + from + (select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no + from + (select ucd.column_value row_data, + dbms_crypto.hash( value(ucd).getclobval(),3) row_hash, + dbms_crypto.hash( extract(value(ucd), :join_by_xpath ).getClobVal(),3/*HASH_SH1*/) pk_hash + from + (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter + from ut_compound_data_tmp ucd + where ucd.data_id = :self_guid + and ucd.item_hash in (select i.item_hash from diff_info i) + ) r, + table( xmlsequence( extract(r.item_data,'/*') ) ) ucd + ) ucd + ) exp + full outer join ( + select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no + from + (select ucd.column_value row_data, + dbms_crypto.hash( value(ucd).getclobval(),3/*HASH_SH1*/) row_hash, + dbms_crypto.hash( extract(value(ucd), :join_by_xpath ).getClobVal(),3/*HASH_SH1*/) pk_hash + from + (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter + from ut_compound_data_tmp ucd + where ucd.data_id = :other_guid + and ucd.item_hash in (select i.item_hash from diff_info i) + ) r, + table( xmlsequence( extract(r.item_data,'/*') ) ) ucd + ) ucd + ) act + on exp.pk_hash = act.pk_hash and exp.duplicate_no = act.duplicate_no + where (exp.row_hash != act.row_hash and act.pk_hash = exp.pk_hash) or + ( + (exp.pk_hash is null or act.pk_hash is null) and + (exp.row_hash is null or act.row_hash is null) + ) + ) + ) + unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:' + ,miss_item as 'Missing:', ext_item as 'Extra:') ) + ) + where rownum <= :max_rows + order by 1, 2]' bulk collect into l_results using a_diff_id, - a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, - a_exclude_xpath, a_include_xpath, a_actual_dataset_guid, + a_join_by_xpath,a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, + a_join_by_xpath,a_exclude_xpath, a_include_xpath, a_actual_dataset_guid, a_max_rows; return l_results; end; @@ -325,8 +333,8 @@ create or replace package body ut_compound_data_helper is coalesce(exp.duplicate_no,act.duplicate_no) duplicate_no, case when act.row_hash is null then - 'miss row' - else 'extra row' + 'Missing:' + else 'Extra:' end diffed_type, case when exp.row_hash is null then xmlserialize(content act.row_data no indent) diff --git a/source/expectations/data_values/ut_compound_data_value.tpb b/source/expectations/data_values/ut_compound_data_value.tpb index cd3af7ac4..6228b31fe 100644 --- a/source/expectations/data_values/ut_compound_data_value.tpb +++ b/source/expectations/data_values/ut_compound_data_value.tpb @@ -155,10 +155,10 @@ create or replace type body ut_compound_data_value as execute immediate 'select count(*) from ' || l_ut_owner || '.ut_compound_data_diff_tmp where diff_id = :diff_id' into l_diff_row_count using l_diff_id; if l_diff_row_count > 0 then - if a_join_by_xpath is not null then + if a_join_by_xpath is not null then l_row_diffs := ut_compound_data_helper.get_rows_diff( self.data_id, l_actual.data_id, l_diff_id, c_max_rows, a_exclude_xpath, a_include_xpath, a_join_by_xpath - ); + ); else l_row_diffs := ut_compound_data_helper.get_rows_diff( self.data_id, l_actual.data_id, l_diff_id, c_max_rows, a_exclude_xpath, a_include_xpath @@ -253,7 +253,7 @@ create or replace type body ut_compound_data_value as l_diff_id := ut_compound_data_helper.get_hash(self.data_id||l_other.data_id); l_column_filter := ut_compound_data_helper.get_columns_filter(a_exclude_xpath, a_include_xpath); - + -- Pre generate hash minus to leave only onese that are diffrent, for example duplicates or diffrent hash execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,duplicate_no ) select :diff_id,x.item_hash,tmp.duplicate_no @@ -278,9 +278,11 @@ create or replace type body ut_compound_data_value as where data_id = :self_guid ))tmp ,ut_compound_data_tmp x - where tmp.item_hash = x.item_hash' + where tmp.item_hash = x.item_hash + and x.data_id in (:self_guid,:other_guid)' using l_diff_id, self.data_id, l_other.data_id - ,l_other.data_id,self.data_id; + ,l_other.data_id,self.data_id, + self.data_id, l_other.data_id; --result is OK only if both are same if sql%rowcount = 0 and self.elements_count = l_other.elements_count then l_result := 0; diff --git a/source/expectations/data_values/ut_data_value_refcursor.tpb b/source/expectations/data_values/ut_data_value_refcursor.tpb index cabe18011..d9edf4132 100644 --- a/source/expectations/data_values/ut_data_value_refcursor.tpb +++ b/source/expectations/data_values/ut_data_value_refcursor.tpb @@ -61,7 +61,7 @@ create or replace type body ut_data_value_refcursor as execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_tmp(data_id, item_no, item_data, item_hash) ' || - 'select :self_guid, :self_row_count + rownum, value(a), dbms_crypto.hash( value(a).getclobval(),3)' || + 'select :self_guid, :self_row_count + rownum, value(a), ut_compound_data_helper.get_hash(value(a).GetClobVal())' || ' from table( xmlsequence( extract(:l_xml,''ROWSET/*'') ) ) a' using in self.data_id, self.elements_count, l_xml; From 3ca16f7bdcb17c03eaad132c7f76d32dd4b359f5 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Thu, 3 May 2018 08:47:29 +0100 Subject: [PATCH 04/24] Updated JoinBy query --- .../data_values/ut_compound_data_helper.pkb | 116 ++++++++++-------- .../data_values/ut_compound_data_value.tpb | 22 ++-- .../data_values/ut_data_value_refcursor.tpb | 10 +- .../test_expectations_cursor.pkb | 73 +++++++++++ .../test_expectations_cursor.pks | 12 ++ 5 files changed, 164 insertions(+), 69 deletions(-) diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index d66c7c80f..df5f5453b 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -167,68 +167,76 @@ create or replace package body ut_compound_data_helper is ) return tt_row_diffs is l_column_filter varchar2(32767); l_results tt_row_diffs; - l_sql varchar2(32767); -- REMOVE LATER also for unorder begin l_column_filter := get_columns_filter(a_exclude_xpath,a_include_xpath); /** * Since its unordered search we cannot select max rows from diffs as we miss some comparision records * We will restrict output on higher level of select - */ + **/ - l_sql := q'[ - with - diff_info as (select item_hash from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) + execute immediate q'[ + with diff_info as (select item_hash from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) select rn,diff_type,diffed_row - from (select dense_rank() over (order by pk_hash) as rn, diff_type,data_item diffed_row - from (select nvl(exp.pk_hash, act.pk_hash) pk_hash, - xmlserialize(content exp.row_data no indent) exp_item, - xmlserialize(content act.row_data no indent) act_item - from - (select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no - from - (select ucd.column_value row_data, - dbms_crypto.hash( value(ucd).getclobval(),3) row_hash, - dbms_crypto.hash( extract(value(ucd),']'|| a_join_by_xpath ||q'[').getClobVal(),3/*HASH_SH1*/) pk_hash - from - (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter - from ut_compound_data_tmp ucd - where ucd.data_id = :self_guid - and ucd.item_hash in (select i.item_hash from diff_info i) - ) r, - table( xmlsequence( extract(r.item_data,'/*') ) ) ucd - ) ucd - ) exp - join ( - select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no - from - (select ucd.column_value row_data, - dbms_crypto.hash( value(ucd).getclobval(),3/*HASH_SH1*/) row_hash, - dbms_crypto.hash( extract(value(ucd),']'|| a_join_by_xpath ||q'[').getClobVal(),3/*HASH_SH1*/) pk_hash - from - (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter - from ut_compound_data_tmp ucd - where ucd.data_id = :other_guid - and ucd.item_hash in (select i.item_hash from diff_info i) - ) r, - table( xmlsequence( extract(r.item_data,'/*') ) ) ucd - ) ucd - ) act - on exp.pk_hash = act.pk_hash and exp.duplicate_no = act.duplicate_no - where exp.row_hash != act.row_hash - or exp.row_hash is null - or act.row_hash is null - ) - unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:') ) - ) - where rownum < :max_rows - order by 1, 2]'; - - execute immediate l_sql + from + (select dense_rank() over (order by pk_hash) as rn, diff_type,data_item diffed_row + from + (select pk_hash + ,case when exp_item is not null and act_item is not null then exp_item else null end exp_item + ,case when exp_item is not null and act_item is not null then act_item else null end act_item + ,case when exp_item is not null and act_item is null then exp_item else null end miss_item + ,case when exp_item is null and act_item is not null then act_item else null end ext_item + from + (select nvl(exp.pk_hash, act.pk_hash) pk_hash, + xmlserialize(content exp.row_data no indent) exp_item, + xmlserialize(content act.row_data no indent) act_item + from + (select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no + from + (select ucd.column_value row_data, + dbms_crypto.hash( value(ucd).getclobval(),3) row_hash, + dbms_crypto.hash( extract(value(ucd), :join_by_xpath ).getClobVal(),3/*HASH_SH1*/) pk_hash + from + (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter + from ut_compound_data_tmp ucd + where ucd.data_id = :self_guid + and ucd.item_hash in (select i.item_hash from diff_info i) + ) r, + table( xmlsequence( extract(r.item_data,'/*') ) ) ucd + ) ucd + ) exp + full outer join ( + select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no + from + (select ucd.column_value row_data, + dbms_crypto.hash( value(ucd).getclobval(),3/*HASH_SH1*/) row_hash, + dbms_crypto.hash( extract(value(ucd), :join_by_xpath ).getClobVal(),3/*HASH_SH1*/) pk_hash + from + (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter + from ut_compound_data_tmp ucd + where ucd.data_id = :other_guid + and ucd.item_hash in (select i.item_hash from diff_info i) + ) r, + table( xmlsequence( extract(r.item_data,'/*') ) ) ucd + ) ucd + ) act + on exp.pk_hash = act.pk_hash and exp.duplicate_no = act.duplicate_no + where (exp.row_hash != act.row_hash and act.pk_hash = exp.pk_hash) or + ( + (exp.pk_hash is null or act.pk_hash is null) and + (exp.row_hash is null or act.row_hash is null) + ) + ) + ) + unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:' + ,miss_item as 'Missing:', ext_item as 'Extra:') ) + ) + where rownum <= :max_rows + order by 1, 2]' bulk collect into l_results using a_diff_id, - a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, - a_exclude_xpath, a_include_xpath, a_actual_dataset_guid, + a_join_by_xpath,a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, + a_join_by_xpath,a_exclude_xpath, a_include_xpath, a_actual_dataset_guid, a_max_rows; return l_results; end; @@ -325,8 +333,8 @@ create or replace package body ut_compound_data_helper is coalesce(exp.duplicate_no,act.duplicate_no) duplicate_no, case when act.row_hash is null then - 'miss row' - else 'extra row' + 'Missing:' + else 'Extra:' end diffed_type, case when exp.row_hash is null then xmlserialize(content act.row_data no indent) diff --git a/source/expectations/data_values/ut_compound_data_value.tpb b/source/expectations/data_values/ut_compound_data_value.tpb index cd3af7ac4..f27abd13b 100644 --- a/source/expectations/data_values/ut_compound_data_value.tpb +++ b/source/expectations/data_values/ut_compound_data_value.tpb @@ -155,10 +155,10 @@ create or replace type body ut_compound_data_value as execute immediate 'select count(*) from ' || l_ut_owner || '.ut_compound_data_diff_tmp where diff_id = :diff_id' into l_diff_row_count using l_diff_id; if l_diff_row_count > 0 then - if a_join_by_xpath is not null then + if a_join_by_xpath is not null then l_row_diffs := ut_compound_data_helper.get_rows_diff( self.data_id, l_actual.data_id, l_diff_id, c_max_rows, a_exclude_xpath, a_include_xpath, a_join_by_xpath - ); + ); else l_row_diffs := ut_compound_data_helper.get_rows_diff( self.data_id, l_actual.data_id, l_diff_id, c_max_rows, a_exclude_xpath, a_include_xpath @@ -253,34 +253,36 @@ create or replace type body ut_compound_data_value as l_diff_id := ut_compound_data_helper.get_hash(self.data_id||l_other.data_id); l_column_filter := ut_compound_data_helper.get_columns_filter(a_exclude_xpath, a_include_xpath); - + -- Pre generate hash minus to leave only onese that are diffrent, for example duplicates or diffrent hash execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,duplicate_no ) select :diff_id,x.item_hash,tmp.duplicate_no from( ( select item_hash,row_number() over (partition by item_hash,data_id order by 1) duplicate_no - from ut_compound_data_tmp + from ' || l_ut_owner || '.ut_compound_data_tmp where data_id = :self_guid minus select item_hash,row_number() over (partition by item_hash,data_id order by 1) - from ut_compound_data_tmp + from ' || l_ut_owner || '.ut_compound_data_tmp where data_id = :other_guid ) union all ( select item_hash,row_number() over (partition by item_hash,data_id order by 1) - from ut_compound_data_tmp + from ' || l_ut_owner || '.ut_compound_data_tmp where data_id = :other_guid minus select item_hash,row_number() over (partition by item_hash,data_id order by 1) - from ut_compound_data_tmp + from ' || l_ut_owner || '.ut_compound_data_tmp where data_id = :self_guid ))tmp - ,ut_compound_data_tmp x - where tmp.item_hash = x.item_hash' + , ' || l_ut_owner || '.ut_compound_data_tmp x + where tmp.item_hash = x.item_hash + and x.data_id in (:self_guid,:other_guid)' using l_diff_id, self.data_id, l_other.data_id - ,l_other.data_id,self.data_id; + ,l_other.data_id,self.data_id, + self.data_id, l_other.data_id; --result is OK only if both are same if sql%rowcount = 0 and self.elements_count = l_other.elements_count then l_result := 0; diff --git a/source/expectations/data_values/ut_data_value_refcursor.tpb b/source/expectations/data_values/ut_data_value_refcursor.tpb index cabe18011..4f0bc437b 100644 --- a/source/expectations/data_values/ut_data_value_refcursor.tpb +++ b/source/expectations/data_values/ut_data_value_refcursor.tpb @@ -15,7 +15,7 @@ create or replace type body ut_data_value_refcursor as See the License for the specific language governing permissions and limitations under the License. */ - + constructor function ut_data_value_refcursor(self in out nocopy ut_data_value_refcursor, a_value sys_refcursor) return self as result is begin init(a_value); @@ -28,8 +28,8 @@ create or replace type body ut_data_value_refcursor as l_ctx number; l_xml xmltype; l_current_date_format varchar2(4000); - l_ut_owner varchar2(250) := ut_utils.ut_owner; cursor_not_open exception; + l_ut_owner varchar2(250) := ut_utils.ut_owner; begin self.is_data_null := ut_utils.boolean_to_int(a_value is null); self.self_type := $$plsql_unit; @@ -58,10 +58,10 @@ create or replace type body ut_data_value_refcursor as loop l_xml := dbms_xmlgen.getxmltype(l_ctx); - - execute immediate + + execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_tmp(data_id, item_no, item_data, item_hash) ' || - 'select :self_guid, :self_row_count + rownum, value(a), dbms_crypto.hash( value(a).getclobval(),3)' || + 'select :self_guid, :self_row_count + rownum, value(a),'||l_ut_owner||'.ut_compound_data_helper.get_hash(value(a).GetClobVal()) ' || ' from table( xmlsequence( extract(:l_xml,''ROWSET/*'') ) ) a' using in self.data_id, self.elements_count, l_xml; diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pkb b/test/core/expectations/compound_data/test_expectations_cursor.pkb index b805105e6..39842f439 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pkb +++ b/test/core/expectations/compound_data/test_expectations_cursor.pkb @@ -1041,5 +1041,78 @@ Rows: [ 2 differences ]% end; end; + procedure cursor_unordered_compare_success is + l_actual SYS_REFCURSOR; + l_expected SYS_REFCURSOR; + begin + --Arrange + open l_actual for select username , user_id from all_users order by username asc; + open l_expected for select username , user_id from all_users order by username desc; + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).unordered; + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + end; + + procedure cursor_unordered_compare_fail is + l_actual SYS_REFCURSOR; + l_expected SYS_REFCURSOR; + l_actual_message varchar2(32767); + l_expected_message varchar2(32767); + begin + --Arrange + open l_actual for select 'test1' username,-100 user_id from dual + union all + select 'test' username,-666 user_id from dual + order by 1 asc; + + open l_expected for select 'test1' username,-100 user_id from dual + union all + select 'test' username,-667 user_id from dual + order by 1 desc; + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).unordered; + l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 2 ]% +Diff:% +Rows: [ 2 differences ] +Missing: test-667% +Extra: test-666%]'; + l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; + --Assert + ut.expect(l_actual_message).to_be_like(l_expected_message); + end; + + procedure cursor_joinby_compare is + l_actual SYS_REFCURSOR; + l_expected SYS_REFCURSOR; + begin + --Arrange + open l_actual for select owner, object_name,object_type from all_objects where owner = user + order by 1,2,3 asc; + open l_expected for select owner, object_name,object_type from all_objects where owner = user + order by 1,2,3 desc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by('OWNER'); + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + end; + + procedure cursor_joinby_compare_twocolumns is + l_actual SYS_REFCURSOR; + l_expected SYS_REFCURSOR; + begin + --Arrange + open l_actual for select owner, object_name,object_type from all_objects where owner = user + order by 1,2,3 asc; + open l_expected for select owner, object_name,object_type from all_objects where owner = user + order by 1,2,3 desc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by(ut3.ut_varchar2_list('OWNER,OBJECT_NAME')); + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + end; + end; / diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pks b/test/core/expectations/compound_data/test_expectations_cursor.pks index 4868b72ce..126c2c7e6 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pks +++ b/test/core/expectations/compound_data/test_expectations_cursor.pks @@ -199,5 +199,17 @@ create or replace package test_expectations_cursor is --%test( Fail on passing implicit column name as exclude filter ) procedure exclude_col_name_implicit; + --%test( Compare cursors using unordered method success) + procedure cursor_unordered_compare_success; + + --%test( Compare cursors using unordered method failure) + procedure cursor_unordered_compare_fail; + + --%test( Compare cursors join by single key ) + procedure cursor_joinby_compare; + + --%test( Compare cursors join by composite key) + procedure cursor_joinby_compare_twocolumns; + end; / From a5e6aaee7e8eefc13e42a54ac45be54e098bc79b Mon Sep 17 00:00:00 2001 From: lwasylow Date: Fri, 4 May 2018 17:17:19 +0100 Subject: [PATCH 05/24] fixes to test name length --- .../expectations/compound_data/test_expectations_cursor.pkb | 4 ++-- .../expectations/compound_data/test_expectations_cursor.pks | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pkb b/test/core/expectations/compound_data/test_expectations_cursor.pkb index 39842f439..feb7f2586 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pkb +++ b/test/core/expectations/compound_data/test_expectations_cursor.pkb @@ -1041,7 +1041,7 @@ Rows: [ 2 differences ]% end; end; - procedure cursor_unordered_compare_success is + procedure cursor_unorderd_compr_success is l_actual SYS_REFCURSOR; l_expected SYS_REFCURSOR; begin @@ -1098,7 +1098,7 @@ Extra: test-666%]'; ut.expect(expectations.failed_expectations_data()).to_be_empty(); end; - procedure cursor_joinby_compare_twocolumns is + procedure cursor_joinby_compare_twocols is l_actual SYS_REFCURSOR; l_expected SYS_REFCURSOR; begin diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pks b/test/core/expectations/compound_data/test_expectations_cursor.pks index 126c2c7e6..688eb2073 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pks +++ b/test/core/expectations/compound_data/test_expectations_cursor.pks @@ -200,7 +200,7 @@ create or replace package test_expectations_cursor is procedure exclude_col_name_implicit; --%test( Compare cursors using unordered method success) - procedure cursor_unordered_compare_success; + procedure cursor_unorderd_compr_success; --%test( Compare cursors using unordered method failure) procedure cursor_unordered_compare_fail; @@ -209,7 +209,7 @@ create or replace package test_expectations_cursor is procedure cursor_joinby_compare; --%test( Compare cursors join by composite key) - procedure cursor_joinby_compare_twocolumns; + procedure cursor_joinby_compare_twocols; end; / From 8a33c25af649c75ad3c1af0dd702750a42f623f4 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Fri, 4 May 2018 20:27:00 +0100 Subject: [PATCH 06/24] Update documentation Update if statement to simplify Remove synonyms for temp tables --- docs/userguide/advanced_data_comparison.md | 24 +++++++++++++++++++ .../create_synonyms_and_grants_for_public.sql | 2 -- .../create_synonyms_and_grants_for_user.sql | 1 - .../data_values/ut_compound_data_value.tpb | 6 ++--- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/docs/userguide/advanced_data_comparison.md b/docs/userguide/advanced_data_comparison.md index e1e5ee36c..74185c3ec 100644 --- a/docs/userguide/advanced_data_comparison.md +++ b/docs/userguide/advanced_data_comparison.md @@ -23,6 +23,9 @@ Advanced data-comparison options are available for the [`equal`](expectations.md - `exclude(a_items varchar2)` - item or comma separated list of items to exclude - `include(a_items ut_varchar2_list)` - table of items to include - `exclude(a_items ut_varchar2_list)` - table of items to exclude + - `unordered` - perform compare on unordered set of data, return only missing or actual + - `join_by(a_columns varchar2)` - columns or comma seperated list of columns to join two cursors by + - `join_by(a_columns ut_varchar2_list)` - table of columns to join two cursors by Each item in the comma separated list can be: - a column name of cursor to be compared @@ -92,6 +95,27 @@ Only the columns 'RN', "A_Column" will be compared. Column 'SOME_COL' is exclude This option can be useful in scenarios where you need to narrow-down the scope of test so that the test is only focused on very specific data. +## Join By option +You can now join two cursors by defining a primary key or composite key that will be used to uniqely identify and compare rows. This option allows us to exactly show which rows are missing, extra and which are diffrent without ordering clause. + +```sql +procedure join_by_username is + l_actual sys_refcursor; + l_expected sys_refcursor; +begin + open l_expected for select username, user_id from all_users + union all + select 'TEST' username, -600 user_id from dual + order by 1 desc; + open l_actual for select username, user_id from all_users + union all + select 'TEST' username, -610 user_id from dual + order by 1 asc; + ut.expect( l_actual ).to_equal( l_expected ).join_by('USERNAME'); +end; +``` +This will show you diffrence in row 'TEST' regardless of order. + ## Defining item as XPath When using XPath expression, keep in mind the following: diff --git a/source/create_synonyms_and_grants_for_public.sql b/source/create_synonyms_and_grants_for_public.sql index 235c6806a..e34df182e 100644 --- a/source/create_synonyms_and_grants_for_public.sql +++ b/source/create_synonyms_and_grants_for_public.sql @@ -142,8 +142,6 @@ create public synonym ut_file_mapping for &&ut3_owner..ut_file_mapping; create public synonym ut_file_mapper for &&ut3_owner..ut_file_mapper; create public synonym ut_key_value_pairs for &&ut3_owner..ut_key_value_pairs; create public synonym ut_key_value_pair for &&ut3_owner..ut_key_value_pair; -create public synonym ut_compound_data_tmp for &&ut3_owner..ut_compound_data_tmp; -create public synonym ut_compound_data_diff_tmp for &&ut3_owner..ut_compound_data_diff_tmp; create public synonym ut_sonar_test_reporter for &&ut3_owner..ut_sonar_test_reporter; begin $if dbms_db_version.version = 12 and dbms_db_version.release >= 2 or dbms_db_version.version > 12 $then diff --git a/source/create_synonyms_and_grants_for_user.sql b/source/create_synonyms_and_grants_for_user.sql index 3044cb32f..ec04a4040 100644 --- a/source/create_synonyms_and_grants_for_user.sql +++ b/source/create_synonyms_and_grants_for_user.sql @@ -164,7 +164,6 @@ create or replace synonym &ut3_user..ut_file_mapper for &&ut3_owner..ut_file_map create or replace synonym &ut3_user..ut_key_value_pairs for &&ut3_owner..ut_key_value_pairs; create or replace synonym &ut3_user..ut_key_value_pair for &&ut3_owner..ut_key_value_pair; create or replace synonym &ut3_user..ut_compound_data_tmp for &&ut3_owner..ut_cursor_data; -create or replace synonym &ut3_user..ut_compound_data_diff_tmp for &&ut3_owner..ut_compound_data_diff_tmp; create or replace synonym &ut3_user..ut_sonar_test_reporter for &&ut3_owner..ut_sonar_test_reporter; begin $if dbms_db_version.version = 12 and dbms_db_version.release >= 2 or dbms_db_version.version > 12 $then diff --git a/source/expectations/data_values/ut_compound_data_value.tpb b/source/expectations/data_values/ut_compound_data_value.tpb index f27abd13b..c1626abc3 100644 --- a/source/expectations/data_values/ut_compound_data_value.tpb +++ b/source/expectations/data_values/ut_compound_data_value.tpb @@ -73,11 +73,9 @@ create or replace type body ut_compound_data_value as overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean := false ) return varchar2 is l_result clob; - l_result_string varchar2(32767); + l_result_string varchar2(32767); begin - if a_join_by_xpath is not null then - l_result := get_data_diff(a_other, a_exclude_xpath, a_include_xpath, a_join_by_xpath); - elsif a_unordered then + if a_unordered then l_result := get_data_diff(a_other, a_exclude_xpath, a_include_xpath, a_unordered); else l_result := get_data_diff(a_other, a_exclude_xpath, a_include_xpath, a_join_by_xpath); From 0fc68d19f85df19fadd8ee3344a64c9314d7541f Mon Sep 17 00:00:00 2001 From: lwasylow Date: Sat, 5 May 2018 10:12:31 +0100 Subject: [PATCH 07/24] Update order by to be done before max row nums are pulled --- .../expectations/data_values/ut_compound_data_diff_tmp.sql | 2 +- .../expectations/data_values/ut_compound_data_helper.pkb | 7 ++++++- source/expectations/data_values/ut_compound_data_tmp.sql | 5 ++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/source/expectations/data_values/ut_compound_data_diff_tmp.sql b/source/expectations/data_values/ut_compound_data_diff_tmp.sql index c10226f2c..4f1385d54 100644 --- a/source/expectations/data_values/ut_compound_data_diff_tmp.sql +++ b/source/expectations/data_values/ut_compound_data_diff_tmp.sql @@ -16,7 +16,7 @@ create global temporary table ut_compound_data_diff_tmp( item_no integer, item_hash raw(128), duplicate_no integer, - --constraint ut_compound_data_diff_tmp_uk1 unique (diff_id, item_no, item_hash, duplicate_no), + constraint ut_compound_data_diff_tmp_uk1 unique (diff_id, item_no, item_hash, duplicate_no), constraint ut_compound_data_diff_tmp_chk check( item_no is not null and item_hash is null and duplicate_no is null or item_no is null and item_hash is not null and duplicate_no is not null diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index df5f5453b..e773f14df 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -178,6 +178,9 @@ create or replace package body ut_compound_data_helper is execute immediate q'[ with diff_info as (select item_hash from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) select rn,diff_type,diffed_row + from + ( + select rn,diff_type,diffed_row from (select dense_rank() over (order by pk_hash) as rn, diff_type,data_item diffed_row from @@ -231,8 +234,10 @@ create or replace package body ut_compound_data_helper is unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:' ,miss_item as 'Missing:', ext_item as 'Extra:') ) ) + order by 1, 2 + ) where rownum <= :max_rows - order by 1, 2]' + ]' bulk collect into l_results using a_diff_id, a_join_by_xpath,a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, diff --git a/source/expectations/data_values/ut_compound_data_tmp.sql b/source/expectations/data_values/ut_compound_data_tmp.sql index 8af8083ef..ed548f0a9 100644 --- a/source/expectations/data_values/ut_compound_data_tmp.sql +++ b/source/expectations/data_values/ut_compound_data_tmp.sql @@ -16,7 +16,6 @@ create global temporary table ut_compound_data_tmp( item_no integer, item_data xmltype, item_hash raw(128), - duplicate_no integer - --constraint ut_cmp_data_tmp_item_ak unique (data_id, item_no), - --constraint ut_cmp_data_tmp_hash_pk unique (data_id, item_hash , duplicate_no) + duplicate_no integer, + constraint ut_cmp_data_tmp_hash_pk unique (data_id,item_no, item_hash , duplicate_no) ) on commit preserve rows; From 7f3d12ed31e8d9fc2cdb3c7aae967381639f7da9 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Sat, 5 May 2018 14:40:14 +0100 Subject: [PATCH 08/24] Updated join by updated docs --- docs/userguide/advanced_data_comparison.md | 53 +++++++- .../data_values/ut_compound_data_diff_tmp.sql | 8 +- .../data_values/ut_compound_data_helper.pkb | 119 ++++++++++-------- .../data_values/ut_compound_data_value.tpb | 53 +++++--- 4 files changed, 155 insertions(+), 78 deletions(-) diff --git a/docs/userguide/advanced_data_comparison.md b/docs/userguide/advanced_data_comparison.md index 74185c3ec..3b0ae2460 100644 --- a/docs/userguide/advanced_data_comparison.md +++ b/docs/userguide/advanced_data_comparison.md @@ -7,7 +7,7 @@ utPLSQL expectations incorporates advanced data comparison options when comparin - nested table and varray Advanced data-comparison options are available for the [`equal`](expectations.md#equal) matcher. - + ## Syntax ``` @@ -58,7 +58,7 @@ Columns 'ignore_me' and "ADate" will get excluded from cursor comparison. The cursor data is equal, when those columns are excluded. This option is useful in scenarios, when you need to exclude incomparable/unpredictable column data like CREATE_DATE of a record that is maintained by default value on a table column. - + ## Selecting columns for data comparison Consider the following example @@ -95,8 +95,51 @@ Only the columns 'RN', "A_Column" will be compared. Column 'SOME_COL' is exclude This option can be useful in scenarios where you need to narrow-down the scope of test so that the test is only focused on very specific data. -## Join By option -You can now join two cursors by defining a primary key or composite key that will be used to uniqely identify and compare rows. This option allows us to exactly show which rows are missing, extra and which are diffrent without ordering clause. +##Unordered + +Unordered option allows for quick comparison of two cursors without need of ordering them in any way. + +Result of such comparison will be limited to only information about row existing or not existing in given set without actual information about exact differences. + + + +```sql +procedure unordered_tst is + l_actual sys_refcursor; + l_expected sys_refcursor; +begin + open l_expected for select username, user_id from all_users + union all + select 'TEST' username, -600 user_id from dual + order by 1 desc; + open l_actual for select username, user_id from all_users + union all + select 'TEST' username, -610 user_id from dual + order by 1 asc; + ut.expect( l_actual ).to_equal( l_expected ).unordered; +end; +``` + + + +Above test will result in two differences of one row extra and one row missing. + + + +```sql + Diff: + Rows: [ 2 differences ] + Missing: TEST-600 + Extra: TEST-610 +``` + + + + + +## Join by option + +You can now join two cursors by defining a primary key or composite key that will be used to uniquely identify and compare rows. This option allows us to exactly show which rows are missing, extra and which are different without ordering clause. ```sql procedure join_by_username is @@ -133,4 +176,4 @@ begin open l_actual for select rownum as rn, 'a' as "A_Column", 'x' SOME_COL, a.* from all_objects a where rownum < 4; ut.expect( l_actual ).to_equal( l_expected ).include( '/ROW/RN|/ROW/A_Column|/ROW/SOME_COL' ); end; -``` +``` diff --git a/source/expectations/data_values/ut_compound_data_diff_tmp.sql b/source/expectations/data_values/ut_compound_data_diff_tmp.sql index 4f1385d54..14d020061 100644 --- a/source/expectations/data_values/ut_compound_data_diff_tmp.sql +++ b/source/expectations/data_values/ut_compound_data_diff_tmp.sql @@ -14,11 +14,13 @@ create global temporary table ut_compound_data_diff_tmp( */ diff_id raw(128), item_no integer, - item_hash raw(128), + pk_hash raw(128), + item_hash raw(128), duplicate_no integer, - constraint ut_compound_data_diff_tmp_uk1 unique (diff_id, item_no, item_hash, duplicate_no), + constraint ut_compound_data_diff_tmp_uk1 unique (diff_id,duplicate_no,item_no,item_hash, pk_hash), constraint ut_compound_data_diff_tmp_chk check( - item_no is not null and item_hash is null and duplicate_no is null + item_no is not null and pk_hash is null and duplicate_no is null or item_no is null and item_hash is not null and duplicate_no is not null + or item_no is null and pk_hash is not null and duplicate_no is not null ) ) on commit preserve rows; diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index e773f14df..e8c1e4e49 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -176,72 +176,85 @@ create or replace package body ut_compound_data_helper is **/ execute immediate q'[ - with diff_info as (select item_hash from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) - select rn,diff_type,diffed_row - from + with diff_info as (select item_hash,pk_hash from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) + select rn,diff_type,diffed_row from ( - select rn,diff_type,diffed_row - from - (select dense_rank() over (order by pk_hash) as rn, diff_type,data_item diffed_row + select diff_type,diffed_row, dense_rank() over (order by pk_hash) rn from + ( + select diff_type,diffed_row,pk_hash from + (select diff_type,data_item diffed_row,pk_hash from - (select pk_hash - ,case when exp_item is not null and act_item is not null then exp_item else null end exp_item - ,case when exp_item is not null and act_item is not null then act_item else null end act_item - ,case when exp_item is not null and act_item is null then exp_item else null end miss_item - ,case when exp_item is null and act_item is not null then act_item else null end ext_item - from - (select nvl(exp.pk_hash, act.pk_hash) pk_hash, + (select nvl(exp.pk_hash, act.pk_hash) pk_hash, xmlserialize(content exp.row_data no indent) exp_item, xmlserialize(content act.row_data no indent) act_item from (select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no from (select ucd.column_value row_data, - dbms_crypto.hash( value(ucd).getclobval(),3) row_hash, - dbms_crypto.hash( extract(value(ucd), :join_by_xpath ).getClobVal(),3/*HASH_SH1*/) pk_hash + r.item_hash row_hash, + r.pk_hash , + ucd.column_value.getRootElement() col_name, + ucd.column_value.getclobval() col_val from - (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter - from ut_compound_data_tmp ucd + (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash + from ut_compound_data_tmp ucd, + diff_info i where ucd.data_id = :self_guid - and ucd.item_hash in (select i.item_hash from diff_info i) + and ucd.item_hash = i.item_hash ) r, - table( xmlsequence( extract(r.item_data,'/*') ) ) ucd + table( xmlsequence( extract(r.item_data,'/*/*') ) ) ucd ) ucd ) exp - full outer join ( + join ( select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no from (select ucd.column_value row_data, - dbms_crypto.hash( value(ucd).getclobval(),3/*HASH_SH1*/) row_hash, - dbms_crypto.hash( extract(value(ucd), :join_by_xpath ).getClobVal(),3/*HASH_SH1*/) pk_hash + r.item_hash row_hash, + r.pk_hash , + ucd.column_value.getRootElement() col_name, + ucd.column_value.getclobval() col_val from - (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter - from ut_compound_data_tmp ucd + (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash + from ut_compound_data_tmp ucd, + diff_info i where ucd.data_id = :other_guid - and ucd.item_hash in (select i.item_hash from diff_info i) + and ucd.item_hash = i.item_hash ) r, - table( xmlsequence( extract(r.item_data,'/*') ) ) ucd + table( xmlsequence( extract(r.item_data,'/*/*') ) ) ucd ) ucd ) act on exp.pk_hash = act.pk_hash and exp.duplicate_no = act.duplicate_no - where (exp.row_hash != act.row_hash and act.pk_hash = exp.pk_hash) or - ( - (exp.pk_hash is null or act.pk_hash is null) and - (exp.row_hash is null or act.row_hash is null) - ) + where dbms_lob.compare(exp.col_val, act.col_val) != 0 ) - ) - unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:' - ,miss_item as 'Missing:', ext_item as 'Extra:') ) + unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:') ) ) - order by 1, 2 - ) - where rownum <= :max_rows + union all + select case when exp.pk_hash is null then 'Extra:' else 'Missing:' end as diff_type, + xmlserialize(content nvl(exp.item_data, act.item_data) no indent) diffed_row, + coalesce(exp.pk_hash,act.pk_hash) pk_hash + from (select extract(ucd.item_data,'/*/*') item_data,i.pk_hash + from ut_compound_data_tmp ucd, + diff_info i + where ucd.data_id = :self_guid + and ucd.item_hash = i.item_hash + ) exp + full outer join ( + select extract(ucd.item_data,'/*/*') item_data,i.pk_hash + from ut_compound_data_tmp ucd, + diff_info i + where ucd.data_id = :other_guid + and ucd.item_hash = i.item_hash + )act + on exp.pk_hash = act.pk_hash + where exp.pk_hash is null or act.pk_hash is null + ) + ) where rn <= :max_rows ]' bulk collect into l_results using a_diff_id, - a_join_by_xpath,a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, - a_join_by_xpath,a_exclude_xpath, a_include_xpath, a_actual_dataset_guid, + a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, + a_exclude_xpath, a_include_xpath, a_actual_dataset_guid, + a_expected_dataset_guid, a_actual_dataset_guid, a_max_rows; return l_results; end; @@ -266,9 +279,10 @@ create or replace package body ut_compound_data_helper is s.column_value.getRootElement() col_name, s.column_value.getclobval() col_val from (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter - from ut_compound_data_tmp ucd + from ut_compound_data_tmp ucd, + diff_info i where ucd.data_id = :self_guid - and ucd.item_no in (select i.item_no from diff_info i) + and ucd.item_no = i.item_no ) r, table( xmlsequence( extract(r.item_data,'/*/*') ) ) s ) exp @@ -277,9 +291,10 @@ create or replace package body ut_compound_data_helper is s.column_value.getRootElement() col_name, s.column_value.getclobval() col_val from (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter - from ut_compound_data_tmp ucd + from ut_compound_data_tmp ucd, + diff_info i where ucd.data_id = :other_guid - and ucd.item_no in (select i.item_no from diff_info i) + and ucd.item_no = i.item_no ) r, table( xmlsequence( extract(r.item_data,'/*/*') ) ) s ) act @@ -348,11 +363,12 @@ create or replace package body ut_compound_data_helper is end diffed_row from (select ucd.*, row_number() over(partition by row_hash order by row_hash) duplicate_no from (select ucd.column_value row_data, - dbms_crypto.hash( value(ucd).getclobval(),3) row_hash - from (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter - from ut_compound_data_tmp ucd + r.item_hash row_hash + from (select ]'||l_column_filter||q'[, ucd.item_no, i.item_hash + from ut_compound_data_tmp ucd, + diff_info i where ucd.data_id = :self_guid - and ucd.item_hash in (select i.item_hash from diff_info i) + and ucd.item_hash = i.item_hash ) r, table( xmlsequence( extract(r.item_data,'/*') ) ) ucd ) ucd @@ -360,11 +376,12 @@ create or replace package body ut_compound_data_helper is full outer join (select ucd.*, row_number() over(partition by row_hash order by row_hash) duplicate_no from (select ucd.column_value row_data, - dbms_crypto.hash( value(ucd).getclobval(),3/*HASH_SH1*/) row_hash - from (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter - from ut_compound_data_tmp ucd + r.item_hash row_hash + from (select ]'||l_column_filter||q'[, ucd.item_no, i.item_hash + from ut_compound_data_tmp ucd, + diff_info i where ucd.data_id = :other_guid - and ucd.item_hash in (select i.item_hash from diff_info i) + and ucd.item_hash = i.item_hash ) r, table( xmlsequence( extract(r.item_data,'/*') ) ) ucd ) ucd diff --git a/source/expectations/data_values/ut_compound_data_value.tpb b/source/expectations/data_values/ut_compound_data_value.tpb index c1626abc3..62b720503 100644 --- a/source/expectations/data_values/ut_compound_data_value.tpb +++ b/source/expectations/data_values/ut_compound_data_value.tpb @@ -150,7 +150,9 @@ create or replace type body ut_compound_data_value as --diff rows and row elements l_diff_id := ut_compound_data_helper.get_hash(self.data_id||l_actual.data_id); -- First tell how many rows are different - execute immediate 'select count(*) from ' || l_ut_owner || '.ut_compound_data_diff_tmp where diff_id = :diff_id' into l_diff_row_count using l_diff_id; + execute immediate 'select count('||case when a_join_by_xpath is not null then 'distinct pk_hash' else '*' end||') from ' + || l_ut_owner || '.ut_compound_data_diff_tmp + where diff_id = :diff_id' into l_diff_row_count using l_diff_id; if l_diff_row_count > 0 then if a_join_by_xpath is not null then @@ -253,34 +255,47 @@ create or replace type body ut_compound_data_value as l_column_filter := ut_compound_data_helper.get_columns_filter(a_exclude_xpath, a_include_xpath); -- Pre generate hash minus to leave only onese that are diffrent, for example duplicates or diffrent hash - execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,duplicate_no ) - select :diff_id,x.item_hash,tmp.duplicate_no + execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no ) + select distinct :diff_id,tmp.item_hash,tmp.pk_hash,tmp.duplicate_no from( ( - select item_hash,row_number() over (partition by item_hash,data_id order by 1) duplicate_no - from ' || l_ut_owner || '.ut_compound_data_tmp + select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no,'|| + case when a_join_by_xpath is null then ':join_by_xpath pk_hash' else + l_ut_owner||'.ut_compound_data_helper.get_hash( extract(value(ucd), :join_by_xpath ).GetClobVal()) pk_hash' + end + ||' from ' || l_ut_owner || '.ut_compound_data_tmp t, + table( xmlsequence( extract(t.item_data,''/*'') ) ) ucd where data_id = :self_guid minus - select item_hash,row_number() over (partition by item_hash,data_id order by 1) - from ' || l_ut_owner || '.ut_compound_data_tmp + select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2),'|| + case when a_join_by_xpath is null then ':join_by_xpath pk_hash' else + l_ut_owner||'.ut_compound_data_helper.get_hash( extract(value(ucd), :join_by_xpath ).GetClobVal()) pk_hash' + end + ||' from ' || l_ut_owner || '.ut_compound_data_tmp t, + table( xmlsequence( extract(t.item_data,''/*'') ) ) ucd where data_id = :other_guid - ) + ) union all ( - select item_hash,row_number() over (partition by item_hash,data_id order by 1) - from ' || l_ut_owner || '.ut_compound_data_tmp + select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2),'|| + case when a_join_by_xpath is null then ':join_by_xpath pk_hash' else + l_ut_owner||'.ut_compound_data_helper.get_hash( extract(value(ucd), :join_by_xpath ).GetClobVal()) pk_hash' + end + ||' from ' || l_ut_owner || '.ut_compound_data_tmp t, + table( xmlsequence( extract(t.item_data,''/*'') ) ) ucd where data_id = :other_guid minus - select item_hash,row_number() over (partition by item_hash,data_id order by 1) - from ' || l_ut_owner || '.ut_compound_data_tmp + select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) ,'|| + case when a_join_by_xpath is null then ':join_by_xpath pk_hash' else + l_ut_owner||'.ut_compound_data_helper.get_hash( extract(value(ucd), :join_by_xpath ).GetClobVal()) pk_hash' + end + ||' from ' || l_ut_owner || '.ut_compound_data_tmp t, + table( xmlsequence( extract(t.item_data,''/*'') ) ) ucd where data_id = :self_guid - ))tmp - , ' || l_ut_owner || '.ut_compound_data_tmp x - where tmp.item_hash = x.item_hash - and x.data_id in (:self_guid,:other_guid)' - using l_diff_id, self.data_id, l_other.data_id - ,l_other.data_id,self.data_id, - self.data_id, l_other.data_id; + ))tmp' + using l_diff_id, + a_join_by_xpath,self.data_id,a_join_by_xpath, l_other.data_id, + a_join_by_xpath,l_other.data_id,a_join_by_xpath,self.data_id; --result is OK only if both are same if sql%rowcount = 0 and self.elements_count = l_other.elements_count then l_result := 0; From b8840a177b7e735975999d36013abf5b2ecfb35e Mon Sep 17 00:00:00 2001 From: lwasylow Date: Sat, 5 May 2018 14:40:14 +0100 Subject: [PATCH 09/24] Updated join by updated docs --- docs/userguide/advanced_data_comparison.md | 53 +++++++- .../data_values/ut_compound_data_diff_tmp.sql | 8 +- .../data_values/ut_compound_data_helper.pkb | 128 ++++++++++-------- .../data_values/ut_compound_data_tmp.sql | 1 + .../data_values/ut_compound_data_value.tpb | 83 +++++++++--- 5 files changed, 191 insertions(+), 82 deletions(-) diff --git a/docs/userguide/advanced_data_comparison.md b/docs/userguide/advanced_data_comparison.md index 74185c3ec..3b0ae2460 100644 --- a/docs/userguide/advanced_data_comparison.md +++ b/docs/userguide/advanced_data_comparison.md @@ -7,7 +7,7 @@ utPLSQL expectations incorporates advanced data comparison options when comparin - nested table and varray Advanced data-comparison options are available for the [`equal`](expectations.md#equal) matcher. - + ## Syntax ``` @@ -58,7 +58,7 @@ Columns 'ignore_me' and "ADate" will get excluded from cursor comparison. The cursor data is equal, when those columns are excluded. This option is useful in scenarios, when you need to exclude incomparable/unpredictable column data like CREATE_DATE of a record that is maintained by default value on a table column. - + ## Selecting columns for data comparison Consider the following example @@ -95,8 +95,51 @@ Only the columns 'RN', "A_Column" will be compared. Column 'SOME_COL' is exclude This option can be useful in scenarios where you need to narrow-down the scope of test so that the test is only focused on very specific data. -## Join By option -You can now join two cursors by defining a primary key or composite key that will be used to uniqely identify and compare rows. This option allows us to exactly show which rows are missing, extra and which are diffrent without ordering clause. +##Unordered + +Unordered option allows for quick comparison of two cursors without need of ordering them in any way. + +Result of such comparison will be limited to only information about row existing or not existing in given set without actual information about exact differences. + + + +```sql +procedure unordered_tst is + l_actual sys_refcursor; + l_expected sys_refcursor; +begin + open l_expected for select username, user_id from all_users + union all + select 'TEST' username, -600 user_id from dual + order by 1 desc; + open l_actual for select username, user_id from all_users + union all + select 'TEST' username, -610 user_id from dual + order by 1 asc; + ut.expect( l_actual ).to_equal( l_expected ).unordered; +end; +``` + + + +Above test will result in two differences of one row extra and one row missing. + + + +```sql + Diff: + Rows: [ 2 differences ] + Missing: TEST-600 + Extra: TEST-610 +``` + + + + + +## Join by option + +You can now join two cursors by defining a primary key or composite key that will be used to uniquely identify and compare rows. This option allows us to exactly show which rows are missing, extra and which are different without ordering clause. ```sql procedure join_by_username is @@ -133,4 +176,4 @@ begin open l_actual for select rownum as rn, 'a' as "A_Column", 'x' SOME_COL, a.* from all_objects a where rownum < 4; ut.expect( l_actual ).to_equal( l_expected ).include( '/ROW/RN|/ROW/A_Column|/ROW/SOME_COL' ); end; -``` +``` diff --git a/source/expectations/data_values/ut_compound_data_diff_tmp.sql b/source/expectations/data_values/ut_compound_data_diff_tmp.sql index 4f1385d54..14d020061 100644 --- a/source/expectations/data_values/ut_compound_data_diff_tmp.sql +++ b/source/expectations/data_values/ut_compound_data_diff_tmp.sql @@ -14,11 +14,13 @@ create global temporary table ut_compound_data_diff_tmp( */ diff_id raw(128), item_no integer, - item_hash raw(128), + pk_hash raw(128), + item_hash raw(128), duplicate_no integer, - constraint ut_compound_data_diff_tmp_uk1 unique (diff_id, item_no, item_hash, duplicate_no), + constraint ut_compound_data_diff_tmp_uk1 unique (diff_id,duplicate_no,item_no,item_hash, pk_hash), constraint ut_compound_data_diff_tmp_chk check( - item_no is not null and item_hash is null and duplicate_no is null + item_no is not null and pk_hash is null and duplicate_no is null or item_no is null and item_hash is not null and duplicate_no is not null + or item_no is null and pk_hash is not null and duplicate_no is not null ) ) on commit preserve rows; diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index e773f14df..83991edb9 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -176,72 +176,85 @@ create or replace package body ut_compound_data_helper is **/ execute immediate q'[ - with diff_info as (select item_hash from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) - select rn,diff_type,diffed_row - from + with diff_info as (select item_hash,pk_hash from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) + select rn,diff_type,diffed_row from ( - select rn,diff_type,diffed_row - from - (select dense_rank() over (order by pk_hash) as rn, diff_type,data_item diffed_row + select diff_type,diffed_row, dense_rank() over (order by pk_hash) rn from + ( + select diff_type,diffed_row,pk_hash from + (select diff_type,data_item diffed_row,pk_hash from - (select pk_hash - ,case when exp_item is not null and act_item is not null then exp_item else null end exp_item - ,case when exp_item is not null and act_item is not null then act_item else null end act_item - ,case when exp_item is not null and act_item is null then exp_item else null end miss_item - ,case when exp_item is null and act_item is not null then act_item else null end ext_item - from - (select nvl(exp.pk_hash, act.pk_hash) pk_hash, + (select nvl(exp.pk_hash, act.pk_hash) pk_hash, xmlserialize(content exp.row_data no indent) exp_item, xmlserialize(content act.row_data no indent) act_item from (select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no from (select ucd.column_value row_data, - dbms_crypto.hash( value(ucd).getclobval(),3) row_hash, - dbms_crypto.hash( extract(value(ucd), :join_by_xpath ).getClobVal(),3/*HASH_SH1*/) pk_hash + r.item_hash row_hash, + r.pk_hash , + ucd.column_value.getRootElement() col_name, + ucd.column_value.getclobval() col_val from - (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter - from ut_compound_data_tmp ucd + (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash + from ut_compound_data_tmp ucd, + diff_info i where ucd.data_id = :self_guid - and ucd.item_hash in (select i.item_hash from diff_info i) + and ucd.item_hash = i.item_hash ) r, - table( xmlsequence( extract(r.item_data,'/*') ) ) ucd + table( xmlsequence( extract(r.item_data,'/*/*') ) ) ucd ) ucd ) exp - full outer join ( + join ( select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no from (select ucd.column_value row_data, - dbms_crypto.hash( value(ucd).getclobval(),3/*HASH_SH1*/) row_hash, - dbms_crypto.hash( extract(value(ucd), :join_by_xpath ).getClobVal(),3/*HASH_SH1*/) pk_hash + r.item_hash row_hash, + r.pk_hash , + ucd.column_value.getRootElement() col_name, + ucd.column_value.getclobval() col_val from - (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter - from ut_compound_data_tmp ucd + (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash + from ut_compound_data_tmp ucd, + diff_info i where ucd.data_id = :other_guid - and ucd.item_hash in (select i.item_hash from diff_info i) + and ucd.item_hash = i.item_hash ) r, - table( xmlsequence( extract(r.item_data,'/*') ) ) ucd + table( xmlsequence( extract(r.item_data,'/*/*') ) ) ucd ) ucd ) act on exp.pk_hash = act.pk_hash and exp.duplicate_no = act.duplicate_no - where (exp.row_hash != act.row_hash and act.pk_hash = exp.pk_hash) or - ( - (exp.pk_hash is null or act.pk_hash is null) and - (exp.row_hash is null or act.row_hash is null) - ) + where dbms_lob.compare(exp.col_val, act.col_val) != 0 ) - ) - unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:' - ,miss_item as 'Missing:', ext_item as 'Extra:') ) + unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:') ) ) - order by 1, 2 - ) - where rownum <= :max_rows + union all + select case when exp.pk_hash is null then 'Extra:' else 'Missing:' end as diff_type, + xmlserialize(content nvl(exp.item_data, act.item_data) no indent) diffed_row, + coalesce(exp.pk_hash,act.pk_hash) pk_hash + from (select extract(ucd.item_data,'/*/*') item_data,i.pk_hash + from ut_compound_data_tmp ucd, + diff_info i + where ucd.data_id = :self_guid + and ucd.item_hash = i.item_hash + ) exp + full outer join ( + select extract(ucd.item_data,'/*/*') item_data,i.pk_hash + from ut_compound_data_tmp ucd, + diff_info i + where ucd.data_id = :other_guid + and ucd.item_hash = i.item_hash + )act + on exp.pk_hash = act.pk_hash + where exp.pk_hash is null or act.pk_hash is null + ) + ) where rn <= :max_rows ]' bulk collect into l_results using a_diff_id, - a_join_by_xpath,a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, - a_join_by_xpath,a_exclude_xpath, a_include_xpath, a_actual_dataset_guid, + a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, + a_exclude_xpath, a_include_xpath, a_actual_dataset_guid, + a_expected_dataset_guid, a_actual_dataset_guid, a_max_rows; return l_results; end; @@ -266,9 +279,10 @@ create or replace package body ut_compound_data_helper is s.column_value.getRootElement() col_name, s.column_value.getclobval() col_val from (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter - from ut_compound_data_tmp ucd + from ut_compound_data_tmp ucd, + diff_info i where ucd.data_id = :self_guid - and ucd.item_no in (select i.item_no from diff_info i) + and ucd.item_no = i.item_no ) r, table( xmlsequence( extract(r.item_data,'/*/*') ) ) s ) exp @@ -277,9 +291,10 @@ create or replace package body ut_compound_data_helper is s.column_value.getRootElement() col_name, s.column_value.getclobval() col_val from (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter - from ut_compound_data_tmp ucd + from ut_compound_data_tmp ucd, + diff_info i where ucd.data_id = :other_guid - and ucd.item_no in (select i.item_no from diff_info i) + and ucd.item_no = i.item_no ) r, table( xmlsequence( extract(r.item_data,'/*/*') ) ) s ) act @@ -328,8 +343,9 @@ create or replace package body ut_compound_data_helper is * Since its unordered search we cannot select max rows from diffs as we miss some comparision records * We will restrict output on higher level of select */ + execute immediate q'[with - diff_info as (select item_hash from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) + diff_info as (select item_hash,duplicate_no from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) select duplicate_no, diffed_type, diffed_row @@ -346,25 +362,29 @@ create or replace package body ut_compound_data_helper is when act.row_hash is null then xmlserialize(content exp.row_data no indent) end diffed_row - from (select ucd.*, row_number() over(partition by row_hash order by row_hash) duplicate_no + from (select ucd.* from (select ucd.column_value row_data, - dbms_crypto.hash( value(ucd).getclobval(),3) row_hash - from (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter - from ut_compound_data_tmp ucd + r.item_hash row_hash, + r.duplicate_no + from (select ]'||l_column_filter||q'[, ucd.item_no, i.item_hash, i.duplicate_no + from ut_compound_data_tmp ucd, + diff_info i where ucd.data_id = :self_guid - and ucd.item_hash in (select i.item_hash from diff_info i) + and ucd.item_hash = i.item_hash ) r, table( xmlsequence( extract(r.item_data,'/*') ) ) ucd ) ucd ) exp full outer join - (select ucd.*, row_number() over(partition by row_hash order by row_hash) duplicate_no + (select ucd.* from (select ucd.column_value row_data, - dbms_crypto.hash( value(ucd).getclobval(),3/*HASH_SH1*/) row_hash - from (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_data item_data_no_filter - from ut_compound_data_tmp ucd + r.item_hash row_hash, + r.duplicate_no + from (select ]'||l_column_filter||q'[, ucd.item_no, i.item_hash, i.duplicate_no + from ut_compound_data_tmp ucd, + diff_info i where ucd.data_id = :other_guid - and ucd.item_hash in (select i.item_hash from diff_info i) + and ucd.item_hash = i.item_hash ) r, table( xmlsequence( extract(r.item_data,'/*') ) ) ucd ) ucd diff --git a/source/expectations/data_values/ut_compound_data_tmp.sql b/source/expectations/data_values/ut_compound_data_tmp.sql index ed548f0a9..827ab9066 100644 --- a/source/expectations/data_values/ut_compound_data_tmp.sql +++ b/source/expectations/data_values/ut_compound_data_tmp.sql @@ -16,6 +16,7 @@ create global temporary table ut_compound_data_tmp( item_no integer, item_data xmltype, item_hash raw(128), + pk_hash raw(128), duplicate_no integer, constraint ut_cmp_data_tmp_hash_pk unique (data_id,item_no, item_hash , duplicate_no) ) on commit preserve rows; diff --git a/source/expectations/data_values/ut_compound_data_value.tpb b/source/expectations/data_values/ut_compound_data_value.tpb index c1626abc3..961481d28 100644 --- a/source/expectations/data_values/ut_compound_data_value.tpb +++ b/source/expectations/data_values/ut_compound_data_value.tpb @@ -150,7 +150,9 @@ create or replace type body ut_compound_data_value as --diff rows and row elements l_diff_id := ut_compound_data_helper.get_hash(self.data_id||l_actual.data_id); -- First tell how many rows are different - execute immediate 'select count(*) from ' || l_ut_owner || '.ut_compound_data_diff_tmp where diff_id = :diff_id' into l_diff_row_count using l_diff_id; + execute immediate 'select count('||case when a_join_by_xpath is not null then 'distinct pk_hash' else '*' end||') from ' + || l_ut_owner || '.ut_compound_data_diff_tmp + where diff_id = :diff_id' into l_diff_row_count using l_diff_id; if l_diff_row_count > 0 then if a_join_by_xpath is not null then @@ -241,7 +243,18 @@ create or replace type body ut_compound_data_value as l_diff_id ut_compound_data_helper.t_hash; l_result integer; l_row_diffs ut_compound_data_helper.tt_row_diffs; - c_max_rows constant integer := 20; + c_max_rows constant integer := 20; + + type t_fetch_pk_hash_rec is record + ( + data_id raw(32), + item_hash raw(128), + item_data xmltype + ); + type t_fetch_pk_hash_tab is table of t_fetch_pk_hash_rec; + + l_fetch_tab t_fetch_pk_hash_tab; + l_pk_hash_cursor sys_refcursor; begin if not a_other is of (ut_compound_data_value) then raise value_error; @@ -252,35 +265,65 @@ create or replace type body ut_compound_data_value as l_diff_id := ut_compound_data_helper.get_hash(self.data_id||l_other.data_id); l_column_filter := ut_compound_data_helper.get_columns_filter(a_exclude_xpath, a_include_xpath); + /** + * Due to incompatibility issues in XML between 11 and 12.2 and 12.1 versions we will prepopulate pk_hash upfront to + * avoid optimizer incorrectly rewrite and causing NULL error or ORA-600 + **/ + + if a_join_by_xpath is not null then + + open l_pk_hash_cursor for q'[select t.data_id,t.item_hash,t.item_data + from ]'|| l_ut_owner ||q'[.ut_compound_data_tmp t + ,xmltable('*' + passing t.item_data + columns + xml_pk_hash clob path ']'|| a_join_by_xpath ||q'[' + ) ucd + where data_id = :self_guid or data_id = :other_guid ]' + using self.data_id, l_other.data_id; + fetch l_pk_hash_cursor bulk collect into l_fetch_tab; + for pks in 1..l_fetch_tab.COUNT + loop + execute immediate 'update '|| l_ut_owner ||'.ut_compound_data_tmp + set pk_hash = '|| l_ut_owner ||'.ut_compound_data_helper.get_hash(extract(:item_data,:join_by_xpath).GetClobVal()) + where item_hash = :item_hash' + using l_fetch_tab(pks).item_data,a_join_by_xpath , l_fetch_tab(pks).item_hash; + end loop; + + close l_pk_hash_cursor; + end if; + + -- Pre generate hash minus to leave only onese that are diffrent, for example duplicates or diffrent hash - execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,duplicate_no ) - select :diff_id,x.item_hash,tmp.duplicate_no + execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no ) + select distinct :diff_id,tmp.item_hash,tmp.pk_hash,tmp.duplicate_no from( ( - select item_hash,row_number() over (partition by item_hash,data_id order by 1) duplicate_no - from ' || l_ut_owner || '.ut_compound_data_tmp + select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no, + pk_hash + from ' || l_ut_owner || '.ut_compound_data_tmp t where data_id = :self_guid minus - select item_hash,row_number() over (partition by item_hash,data_id order by 1) - from ' || l_ut_owner || '.ut_compound_data_tmp + select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no, + pk_hash + from ' || l_ut_owner || '.ut_compound_data_tmp t where data_id = :other_guid - ) + ) union all ( - select item_hash,row_number() over (partition by item_hash,data_id order by 1) - from ' || l_ut_owner || '.ut_compound_data_tmp + select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no, + pk_hash + from ' || l_ut_owner || '.ut_compound_data_tmp t where data_id = :other_guid minus - select item_hash,row_number() over (partition by item_hash,data_id order by 1) - from ' || l_ut_owner || '.ut_compound_data_tmp + select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no, + pk_hash + from ' || l_ut_owner || '.ut_compound_data_tmp t where data_id = :self_guid - ))tmp - , ' || l_ut_owner || '.ut_compound_data_tmp x - where tmp.item_hash = x.item_hash - and x.data_id in (:self_guid,:other_guid)' - using l_diff_id, self.data_id, l_other.data_id - ,l_other.data_id,self.data_id, - self.data_id, l_other.data_id; + ))tmp' + using l_diff_id, + self.data_id, l_other.data_id, + l_other.data_id,self.data_id; --result is OK only if both are same if sql%rowcount = 0 and self.elements_count = l_other.elements_count then l_result := 0; From aaa0cc465eb94ab4769d49a489c5f108a5964df4 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Mon, 7 May 2018 08:14:56 +0100 Subject: [PATCH 10/24] Update to sort random order cause of optimizer Capture PK Hash to improve lookup --- docs/userguide/advanced_data_comparison.md | 12 +- .../data_values/ut_compound_data_diff_tmp.sql | 3 +- .../data_values/ut_compound_data_helper.pkb | 34 +++-- .../data_values/ut_compound_data_helper.pks | 3 +- .../data_values/ut_compound_data_tmp.sql | 1 - .../data_values/ut_compound_data_value.tpb | 129 +++++++++++------- .../test_expectations_cursor.pkb | 15 +- .../test_expectations_cursor.pks | 3 + 8 files changed, 134 insertions(+), 66 deletions(-) diff --git a/docs/userguide/advanced_data_comparison.md b/docs/userguide/advanced_data_comparison.md index 3b0ae2460..e003b0c47 100644 --- a/docs/userguide/advanced_data_comparison.md +++ b/docs/userguide/advanced_data_comparison.md @@ -137,7 +137,7 @@ Above test will result in two differences of one row extra and one row missing. -## Join by option +## Join By option You can now join two cursors by defining a primary key or composite key that will be used to uniquely identify and compare rows. This option allows us to exactly show which rows are missing, extra and which are different without ordering clause. @@ -157,7 +157,15 @@ begin ut.expect( l_actual ).to_equal( l_expected ).join_by('USERNAME'); end; ``` -This will show you diffrence in row 'TEST' regardless of order. +This will show you difference in row 'TEST' regardless of order. + +```sql + Rows: [ 1 differences ] + Expected: -600 for key: TEST + Actual: -610 for key: TEST +``` + +**Please note that .join_by option will take longer to process due to need of parsing via primary keys.** ## Defining item as XPath When using XPath expression, keep in mind the following: diff --git a/source/expectations/data_values/ut_compound_data_diff_tmp.sql b/source/expectations/data_values/ut_compound_data_diff_tmp.sql index 14d020061..95cb2f1f0 100644 --- a/source/expectations/data_values/ut_compound_data_diff_tmp.sql +++ b/source/expectations/data_values/ut_compound_data_diff_tmp.sql @@ -15,7 +15,8 @@ create global temporary table ut_compound_data_diff_tmp( diff_id raw(128), item_no integer, pk_hash raw(128), - item_hash raw(128), + pk_value varchar2(4000), + item_hash raw(128), duplicate_no integer, constraint ut_compound_data_diff_tmp_uk1 unique (diff_id,duplicate_no,item_no,item_hash, pk_hash), constraint ut_compound_data_diff_tmp_chk check( diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index 83991edb9..1b22386dd 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -176,15 +176,15 @@ create or replace package body ut_compound_data_helper is **/ execute immediate q'[ - with diff_info as (select item_hash,pk_hash from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) - select rn,diff_type,diffed_row from + with diff_info as (select item_hash,pk_hash,pk_value from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) + select rn,diff_type,diffed_row,pk_value from ( - select diff_type,diffed_row, dense_rank() over (order by pk_hash) rn from + select diff_type,diffed_row, dense_rank() over (order by pk_hash) rn,pk_value from ( - select diff_type,diffed_row,pk_hash from - (select diff_type,data_item diffed_row,pk_hash + select diff_type,diffed_row,pk_hash,pk_value from + (select diff_type,data_item diffed_row,pk_hash,pk_value from - (select nvl(exp.pk_hash, act.pk_hash) pk_hash, + (select nvl(exp.pk_hash, act.pk_hash) pk_hash,nvl(exp.pk_value, act.pk_value) pk_value, xmlserialize(content exp.row_data no indent) exp_item, xmlserialize(content act.row_data no indent) act_item from @@ -193,10 +193,11 @@ create or replace package body ut_compound_data_helper is (select ucd.column_value row_data, r.item_hash row_hash, r.pk_hash , + r.pk_value, ucd.column_value.getRootElement() col_name, ucd.column_value.getclobval() col_val from - (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash + (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash, i.pk_value from ut_compound_data_tmp ucd, diff_info i where ucd.data_id = :self_guid @@ -211,10 +212,11 @@ create or replace package body ut_compound_data_helper is (select ucd.column_value row_data, r.item_hash row_hash, r.pk_hash , + r.pk_value, ucd.column_value.getRootElement() col_name, ucd.column_value.getclobval() col_val from - (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash + (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash, i.pk_value from ut_compound_data_tmp ucd, diff_info i where ucd.data_id = :other_guid @@ -231,7 +233,8 @@ create or replace package body ut_compound_data_helper is union all select case when exp.pk_hash is null then 'Extra:' else 'Missing:' end as diff_type, xmlserialize(content nvl(exp.item_data, act.item_data) no indent) diffed_row, - coalesce(exp.pk_hash,act.pk_hash) pk_hash + coalesce(exp.pk_hash,act.pk_hash) pk_hash, + null pk_value from (select extract(ucd.item_data,'/*/*') item_data,i.pk_hash from ut_compound_data_tmp ucd, diff_info i @@ -269,9 +272,12 @@ create or replace package body ut_compound_data_helper is l_column_filter := get_columns_filter(a_exclude_xpath,a_include_xpath); execute immediate q'[ with - diff_info as (select item_no from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid and rownum <= :max_rows) + diff_info as ( select item_no + from + (select item_no from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid order by item_no asc) + where rownum <= :max_rows) select * - from (select rn, diff_type, xmlserialize(content data_item no indent) diffed_row + from (select rn, diff_type, xmlserialize(content data_item no indent) diffed_row, null pk_value from (select nvl(exp.rn, act.rn) rn, xmlagg(exp.col order by exp.col_no) exp_item, xmlagg(act.col order by act.col_no) act_item @@ -307,7 +313,8 @@ create or replace package body ut_compound_data_helper is union all select nvl(exp.item_no, act.item_no) rn, case when exp.item_no is null then 'Extra:' else 'Missing:' end as diff_type, - xmlserialize(content nvl(exp.item_data, act.item_data) no indent) diffed_row + xmlserialize(content nvl(exp.item_data, act.item_data) no indent) diffed_row, + null pk_value from (select ucd.item_no, extract(ucd.item_data,'/*/*') item_data from ut_compound_data_tmp ucd where ucd.data_id = :self_guid @@ -348,7 +355,8 @@ create or replace package body ut_compound_data_helper is diff_info as (select item_hash,duplicate_no from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) select duplicate_no, diffed_type, - diffed_row + diffed_row, + null pk_value from (select coalesce(exp.duplicate_no,act.duplicate_no) duplicate_no, diff --git a/source/expectations/data_values/ut_compound_data_helper.pks b/source/expectations/data_values/ut_compound_data_helper.pks index 3f8595233..d71af802d 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pks +++ b/source/expectations/data_values/ut_compound_data_helper.pks @@ -31,7 +31,8 @@ create or replace package ut_compound_data_helper authid definer is type t_row_diffs is record( rn integer, diff_type varchar2(250), - diffed_row clob + diffed_row clob, + pk_value varchar2(4000) ); type tt_row_diffs is table of t_row_diffs; diff --git a/source/expectations/data_values/ut_compound_data_tmp.sql b/source/expectations/data_values/ut_compound_data_tmp.sql index 827ab9066..ed548f0a9 100644 --- a/source/expectations/data_values/ut_compound_data_tmp.sql +++ b/source/expectations/data_values/ut_compound_data_tmp.sql @@ -16,7 +16,6 @@ create global temporary table ut_compound_data_tmp( item_no integer, item_data xmltype, item_hash raw(128), - pk_hash raw(128), duplicate_no integer, constraint ut_cmp_data_tmp_hash_pk unique (data_id,item_no, item_hash , duplicate_no) ) on commit preserve rows; diff --git a/source/expectations/data_values/ut_compound_data_value.tpb b/source/expectations/data_values/ut_compound_data_value.tpb index 961481d28..ce5767e21 100644 --- a/source/expectations/data_values/ut_compound_data_value.tpb +++ b/source/expectations/data_values/ut_compound_data_value.tpb @@ -139,6 +139,7 @@ create or replace type body ut_compound_data_value as l_actual ut_compound_data_value; l_diff_id ut_compound_data_helper.t_hash; l_row_diffs ut_compound_data_helper.tt_row_diffs; + begin if not a_other is of (ut_compound_data_value) then raise value_error; @@ -173,7 +174,11 @@ create or replace type body ut_compound_data_value as ut_utils.append_to_clob( l_result, l_message ); for i in 1 .. l_row_diffs.count loop l_results.extend; - l_results(l_results.last) := ' Row No. '||l_row_diffs(i).rn||' - '||rpad(l_row_diffs(i).diff_type,10)||l_row_diffs(i).diffed_row; + if a_join_by_xpath is not null and l_row_diffs(i).pk_value is not null then + l_results(l_results.last) := ' '||rpad(l_row_diffs(i).diff_type,10)||l_row_diffs(i).diffed_row||' for key: '||l_row_diffs(i).pk_value; + else + l_results(l_results.last) := ' Row No. '||l_row_diffs(i).rn||' - '||rpad(l_row_diffs(i).diff_type,10)||l_row_diffs(i).diffed_row; + end if; end loop; ut_utils.append_to_clob(l_result,l_results); end if; @@ -245,16 +250,28 @@ create or replace type body ut_compound_data_value as l_row_diffs ut_compound_data_helper.tt_row_diffs; c_max_rows constant integer := 20; - type t_fetch_pk_hash_rec is record + type t_pk_val_rec is record ( - data_id raw(32), - item_hash raw(128), - item_data xmltype + data_id raw(32), + item_hash raw(128), + pk_hash raw(128), + pk_value varchar2(4000) ); - type t_fetch_pk_hash_tab is table of t_fetch_pk_hash_rec; - l_fetch_tab t_fetch_pk_hash_tab; - l_pk_hash_cursor sys_refcursor; + type t_pk_val_tab is table of t_pk_val_rec; + l_pk_val_tab t_pk_val_tab; + + function get_column_xpath(a_join_by_xpath varchar2) return varchar2 is + l_column varchar2(32767); + begin + if a_join_by_xpath is not null then + l_column := l_ut_owner ||'.ut_compound_data_helper.get_hash(extract(t.item_data,:join_by_xpath).GetClobVal()) pk_hash '; + else + l_column := ':join_by_xpath pk_hash '; + end if; + return l_column; + end; + begin if not a_other is of (ut_compound_data_value) then raise value_error; @@ -268,60 +285,78 @@ create or replace type body ut_compound_data_value as /** * Due to incompatibility issues in XML between 11 and 12.2 and 12.1 versions we will prepopulate pk_hash upfront to * avoid optimizer incorrectly rewrite and causing NULL error or ORA-600 - **/ + **/ + -- Pre generate hash minus to leave only onese that are diffrent, for example duplicates or diffrent hash + /*if a_join_by_xpath is not null then + execute immediate q'[select + data_id,item_hash,pk_hash, + listagg(extractvalue(tcd.column_value, '/*'), '; ') within GROUP(ORDER BY item_hash) pk_value + from + (select + ucd.column_value row_data, + ]'|| l_ut_owner ||q'[.ut_compound_data_helper.get_hash(extract(ucd.column_value,:join_xpath).GetClobVal()) pk_hash , + t.item_hash, + t.data_id + from ]' || l_ut_owner || q'[.ut_compound_data_tmp t , + table(xmlsequence(extract(t.item_data,'/*'))) ucd + where data_id = :self_guid or data_id = :other_guid ), + table(xmlsequence(extract(row_data ,:join_xpath))) tcd + group by pk_hash,item_hash,data_id]' + bulk collect into l_pk_val_tab using a_join_by_xpath,self.data_id, l_other.data_id,a_join_by_xpath; - if a_join_by_xpath is not null then - - open l_pk_hash_cursor for q'[select t.data_id,t.item_hash,t.item_data - from ]'|| l_ut_owner ||q'[.ut_compound_data_tmp t - ,xmltable('*' - passing t.item_data - columns - xml_pk_hash clob path ']'|| a_join_by_xpath ||q'[' - ) ucd - where data_id = :self_guid or data_id = :other_guid ]' - using self.data_id, l_other.data_id; - fetch l_pk_hash_cursor bulk collect into l_fetch_tab; - for pks in 1..l_fetch_tab.COUNT - loop - execute immediate 'update '|| l_ut_owner ||'.ut_compound_data_tmp - set pk_hash = '|| l_ut_owner ||'.ut_compound_data_helper.get_hash(extract(:item_data,:join_by_xpath).GetClobVal()) - where item_hash = :item_hash' - using l_fetch_tab(pks).item_data,a_join_by_xpath , l_fetch_tab(pks).item_hash; - end loop; - - close l_pk_hash_cursor; + forall pk_vals in 1..l_pk_val_tab.COUNT + update ut_compound_data_tmp + set pk_hash = l_pk_val_tab(pk_vals).pk_hash + ,pk_value = l_pk_val_tab(pk_vals).pk_value + where data_id = l_pk_val_tab(pk_vals).data_id + and item_hash = l_pk_val_tab(pk_vals).item_hash; end if; - - - -- Pre generate hash minus to leave only onese that are diffrent, for example duplicates or diffrent hash - execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no ) + */ + + execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no) + with calc_pk as + ( select data_id,item_hash, '||get_column_xpath(a_join_by_xpath)||' + from ' || l_ut_owner || '.ut_compound_data_tmp t + ) select distinct :diff_id,tmp.item_hash,tmp.pk_hash,tmp.duplicate_no from( ( select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no, - pk_hash - from ' || l_ut_owner || '.ut_compound_data_tmp t - where data_id = :self_guid + pc.pk_hash + from ' || l_ut_owner || '.ut_compound_data_tmp t, + calc_pk pc + where t.data_id = :self_guid + and pc.data_id = t.data_id + and pc.item_hash = t.item_hash minus select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no, - pk_hash - from ' || l_ut_owner || '.ut_compound_data_tmp t - where data_id = :other_guid + pc.pk_hash + from ' || l_ut_owner || '.ut_compound_data_tmp t, + calc_pk pc + where t.data_id = :other_guid + and pc.data_id = t.data_id + and pc.item_hash = t.item_hash ) union all ( select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no, - pk_hash - from ' || l_ut_owner || '.ut_compound_data_tmp t - where data_id = :other_guid + pc.pk_hash + from ' || l_ut_owner || '.ut_compound_data_tmp t, + calc_pk pc + where t.data_id = :other_guid + and pc.data_id = t.data_id + and pc.item_hash = t.item_hash minus select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no, - pk_hash - from ' || l_ut_owner || '.ut_compound_data_tmp t - where data_id = :self_guid + pc.pk_hash + from ' || l_ut_owner || '.ut_compound_data_tmp t, + calc_pk pc + where t.data_id = :self_guid + and pc.data_id = t.data_id + and pc.item_hash = t.item_hash ))tmp' - using l_diff_id, + using a_join_by_xpath, + l_diff_id, self.data_id, l_other.data_id, l_other.data_id,self.data_id; --result is OK only if both are same diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pkb b/test/core/expectations/compound_data/test_expectations_cursor.pkb index feb7f2586..7f9e3986a 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pkb +++ b/test/core/expectations/compound_data/test_expectations_cursor.pkb @@ -1113,6 +1113,19 @@ Extra: test-666%]'; --Assert ut.expect(expectations.failed_expectations_data()).to_be_empty(); end; - + + procedure cursor_joinby_compare_1000 is + l_actual SYS_REFCURSOR; + l_expected SYS_REFCURSOR; + begin + --Arrange + open l_actual for select object_name from all_objects where rownum <=1100; + open l_expected for select object_name from all_objects where rownum <=1100; + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by('OBJECT_NAME'); + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + end; + end; / diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pks b/test/core/expectations/compound_data/test_expectations_cursor.pks index 688eb2073..3447b4a15 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pks +++ b/test/core/expectations/compound_data/test_expectations_cursor.pks @@ -211,5 +211,8 @@ create or replace package test_expectations_cursor is --%test( Compare cursors join by composite key) procedure cursor_joinby_compare_twocols; + --%test( Compare cursors join by single key more than 1000 rows) + procedure cursor_joinby_compare_1000; + end; / From 784c509bb9ae602419f6af2d5e8d024caddb75e4 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Wed, 9 May 2018 19:52:27 +0100 Subject: [PATCH 11/24] Update pk join cursor fix to get columns diff --- .../data_values/ut_compound_data_helper.pkb | 33 ++++- .../data_values/ut_compound_data_tmp.sql | 1 + .../data_values/ut_compound_data_value.tpb | 131 ++++++++---------- .../data_values/ut_data_value_refcursor.tpb | 4 +- .../test_expectations_cursor.pkb | 98 +++++++++++++ .../test_expectations_cursor.pks | 21 +++ 6 files changed, 210 insertions(+), 78 deletions(-) diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index 1b22386dd..85bc9fe2b 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -86,6 +86,32 @@ create or replace package body ut_compound_data_helper is end if; return l_filter; end; + + /** + * Current get column filter shaving off ROW tag during extract, this not working well with include and XMLTABLE option + * so when there is extract we artificially inject removed tag + **/ + function get_columns_row_filter( + a_exclude_xpath varchar2, a_include_xpath varchar2, + a_table_alias varchar2 := 'ucd', a_column_alias varchar2 := 'item_data' + ) return varchar2 is + l_filter varchar2(32767); + l_source_column varchar2(500) := a_table_alias||'.'||a_column_alias; + begin + -- this SQL statement is constructed in a way that we always get the same number and ordering of substitution variables + -- That is, we always get: l_exclude_xpath, l_include_xpath + -- regardless if the variables are NULL (not to be used) or NOT NULL and will be used for filtering + if a_exclude_xpath is null and a_include_xpath is null then + l_filter := ':l_exclude_xpath, :l_include_xpath, '||l_source_column||' as '||a_column_alias; + elsif a_exclude_xpath is not null and a_include_xpath is null then + l_filter := 'deletexml( '||l_source_column||', :l_exclude_xpath ) as '||a_column_alias||', :l_include_xpath'; + elsif a_exclude_xpath is null and a_include_xpath is not null then + l_filter := ':l_exclude_xpath, xmlelement("ROW",extract( '||l_source_column||', :l_include_xpath )) as '||a_column_alias; + elsif a_exclude_xpath is not null and a_include_xpath is not null then + l_filter := 'xmlelement("ROW",extract( deletexml( '||l_source_column||', :l_exclude_xpath ), :l_include_xpath )) as '||a_column_alias; + end if; + return l_filter; + end; function get_columns_diff( a_expected xmltype, a_actual xmltype, a_exclude_xpath varchar2, a_include_xpath varchar2 @@ -94,7 +120,7 @@ create or replace package body ut_compound_data_helper is l_sql varchar2(32767); l_results tt_column_diffs; begin - l_column_filter := get_columns_filter(a_exclude_xpath, a_include_xpath); + l_column_filter := get_columns_row_filter(a_exclude_xpath, a_include_xpath); l_sql := q'[ with expected_cols as ( select :a_expected as item_data from dual ), @@ -157,6 +183,7 @@ create or replace package body ut_compound_data_helper is execute immediate l_sql bulk collect into l_results using a_expected, a_actual, a_exclude_xpath, a_include_xpath, a_exclude_xpath, a_include_xpath; + return l_results; end; @@ -349,8 +376,8 @@ create or replace package body ut_compound_data_helper is /** * Since its unordered search we cannot select max rows from diffs as we miss some comparision records * We will restrict output on higher level of select - */ - + */ + execute immediate q'[with diff_info as (select item_hash,duplicate_no from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) select duplicate_no, diff --git a/source/expectations/data_values/ut_compound_data_tmp.sql b/source/expectations/data_values/ut_compound_data_tmp.sql index ed548f0a9..827ab9066 100644 --- a/source/expectations/data_values/ut_compound_data_tmp.sql +++ b/source/expectations/data_values/ut_compound_data_tmp.sql @@ -16,6 +16,7 @@ create global temporary table ut_compound_data_tmp( item_no integer, item_data xmltype, item_hash raw(128), + pk_hash raw(128), duplicate_no integer, constraint ut_cmp_data_tmp_hash_pk unique (data_id,item_no, item_hash , duplicate_no) ) on commit preserve rows; diff --git a/source/expectations/data_values/ut_compound_data_value.tpb b/source/expectations/data_values/ut_compound_data_value.tpb index ce5767e21..4732df286 100644 --- a/source/expectations/data_values/ut_compound_data_value.tpb +++ b/source/expectations/data_values/ut_compound_data_value.tpb @@ -250,24 +250,32 @@ create or replace type body ut_compound_data_value as l_row_diffs ut_compound_data_helper.tt_row_diffs; c_max_rows constant integer := 20; - type t_pk_val_rec is record - ( - data_id raw(32), - item_hash raw(128), - pk_hash raw(128), - pk_value varchar2(4000) - ); - - type t_pk_val_tab is table of t_pk_val_rec; - l_pk_val_tab t_pk_val_tab; - - function get_column_xpath(a_join_by_xpath varchar2) return varchar2 is + function get_pk_value(a_join_by_xpath varchar2) return varchar2 is l_column varchar2(32767); begin + /* due to possibility of key being to columns we cannot use xmlextractvalue + usage of xmlagg is possible however it greatly complicates code and performance is impacted. + xpath to be looked at or regex + */ if a_join_by_xpath is not null then - l_column := l_ut_owner ||'.ut_compound_data_helper.get_hash(extract(t.item_data,:join_by_xpath).GetClobVal()) pk_hash '; + l_column := ' extract(t.item_data,:join_by_xpath).GetStringVal() pk_value '; else - l_column := ':join_by_xpath pk_hash '; + l_column := ' :join_by_xpath pk_value'; + end if; + return l_column; + end; + + function get_column_pk_hash(a_join_by_xpath varchar2) return varchar2 is + l_column varchar2(32767); + begin + /* due to possibility of key being to columns we cannot use xmlextractvalue + usage of xmlagg is possible however it greatly complicates code and performance is impacted. + xpath to be looked at or regex + */ + if a_join_by_xpath is not null then + l_column := l_ut_owner ||'.ut_compound_data_helper.get_hash(extract(ucd.item_data,:join_by_xpath).GetClobVal()) pk_hash'; + else + l_column := ':join_by_xpath pk_hash'; end if; return l_column; end; @@ -281,81 +289,58 @@ create or replace type body ut_compound_data_value as l_diff_id := ut_compound_data_helper.get_hash(self.data_id||l_other.data_id); l_column_filter := ut_compound_data_helper.get_columns_filter(a_exclude_xpath, a_include_xpath); - + /** * Due to incompatibility issues in XML between 11 and 12.2 and 12.1 versions we will prepopulate pk_hash upfront to * avoid optimizer incorrectly rewrite and causing NULL error or ORA-600 - **/ - -- Pre generate hash minus to leave only onese that are diffrent, for example duplicates or diffrent hash - /*if a_join_by_xpath is not null then - execute immediate q'[select - data_id,item_hash,pk_hash, - listagg(extractvalue(tcd.column_value, '/*'), '; ') within GROUP(ORDER BY item_hash) pk_value - from - (select - ucd.column_value row_data, - ]'|| l_ut_owner ||q'[.ut_compound_data_helper.get_hash(extract(ucd.column_value,:join_xpath).GetClobVal()) pk_hash , - t.item_hash, - t.data_id - from ]' || l_ut_owner || q'[.ut_compound_data_tmp t , - table(xmlsequence(extract(t.item_data,'/*'))) ucd - where data_id = :self_guid or data_id = :other_guid ), - table(xmlsequence(extract(row_data ,:join_xpath))) tcd - group by pk_hash,item_hash,data_id]' - bulk collect into l_pk_val_tab using a_join_by_xpath,self.data_id, l_other.data_id,a_join_by_xpath; + **/ + execute immediate 'merge into ' || l_ut_owner || '.ut_compound_data_tmp tgt + using ( + select '||l_ut_owner ||'.ut_compound_data_helper.get_hash(ucd.item_data.getclobval()) item_hash, + pk_hash, ucd.item_no, ucd.data_id + from + ( + select '||l_column_filter||','||get_column_pk_hash(a_join_by_xpath)||', item_no, data_id + from ' || l_ut_owner || q'[.ut_compound_data_tmp ucd + where data_id = :self_guid or data_id = :other_guid + ) ucd + ) src + on (tgt.item_no = src.item_no and tgt.data_id = src.data_id) + when matched then update + set tgt.item_hash = src.item_hash, + tgt.pk_hash = src.pk_hash ]' + using a_exclude_xpath, a_include_xpath,a_join_by_xpath,self.data_id, l_other.data_id; - forall pk_vals in 1..l_pk_val_tab.COUNT - update ut_compound_data_tmp - set pk_hash = l_pk_val_tab(pk_vals).pk_hash - ,pk_value = l_pk_val_tab(pk_vals).pk_value - where data_id = l_pk_val_tab(pk_vals).data_id - and item_hash = l_pk_val_tab(pk_vals).item_hash; - end if; - */ - - execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no) - with calc_pk as - ( select data_id,item_hash, '||get_column_xpath(a_join_by_xpath)||' - from ' || l_ut_owner || '.ut_compound_data_tmp t - ) - select distinct :diff_id,tmp.item_hash,tmp.pk_hash,tmp.duplicate_no + /* Peform minus on two sets two get diffrences that will be used later on to print results */ + execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no,pk_value) + with source_data as + ( select t.data_id,t.item_hash,row_number() over (partition by t.pk_hash,t.item_hash,t.data_id order by 1,2) duplicate_no, + pk_hash, '||get_pk_value(a_join_by_xpath)||' + from ' || l_ut_owner || '.ut_compound_data_tmp t + where data_id = :self_guid or data_id = :other_guid + ) + select distinct :diff_id,tmp.item_hash,tmp.pk_hash,tmp.duplicate_no,pk_value from( ( - select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no, - pc.pk_hash - from ' || l_ut_owner || '.ut_compound_data_tmp t, - calc_pk pc + select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value + from source_data t where t.data_id = :self_guid - and pc.data_id = t.data_id - and pc.item_hash = t.item_hash minus - select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no, - pc.pk_hash - from ' || l_ut_owner || '.ut_compound_data_tmp t, - calc_pk pc + select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value + from source_data t where t.data_id = :other_guid - and pc.data_id = t.data_id - and pc.item_hash = t.item_hash ) union all ( - select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no, - pc.pk_hash - from ' || l_ut_owner || '.ut_compound_data_tmp t, - calc_pk pc + select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value + from source_data t where t.data_id = :other_guid - and pc.data_id = t.data_id - and pc.item_hash = t.item_hash minus - select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no, - pc.pk_hash - from ' || l_ut_owner || '.ut_compound_data_tmp t, - calc_pk pc + select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value + from source_data t where t.data_id = :self_guid - and pc.data_id = t.data_id - and pc.item_hash = t.item_hash ))tmp' - using a_join_by_xpath, + using a_join_by_xpath,self.data_id, l_other.data_id, l_diff_id, self.data_id, l_other.data_id, l_other.data_id,self.data_id; diff --git a/source/expectations/data_values/ut_data_value_refcursor.tpb b/source/expectations/data_values/ut_data_value_refcursor.tpb index 4f0bc437b..480aba30c 100644 --- a/source/expectations/data_values/ut_data_value_refcursor.tpb +++ b/source/expectations/data_values/ut_data_value_refcursor.tpb @@ -60,8 +60,8 @@ create or replace type body ut_data_value_refcursor as l_xml := dbms_xmlgen.getxmltype(l_ctx); execute immediate - 'insert into ' || l_ut_owner || '.ut_compound_data_tmp(data_id, item_no, item_data, item_hash) ' || - 'select :self_guid, :self_row_count + rownum, value(a),'||l_ut_owner||'.ut_compound_data_helper.get_hash(value(a).GetClobVal()) ' || + 'insert into ' || l_ut_owner || '.ut_compound_data_tmp(data_id, item_no, item_data) ' || + 'select :self_guid, :self_row_count + rownum, value(a) ' || ' from table( xmlsequence( extract(:l_xml,''ROWSET/*'') ) ) a' using in self.data_id, self.elements_count, l_xml; diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pkb b/test/core/expectations/compound_data/test_expectations_cursor.pkb index 7f9e3986a..5a0f429cc 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pkb +++ b/test/core/expectations/compound_data/test_expectations_cursor.pkb @@ -1127,5 +1127,103 @@ Extra: test-666%]'; ut.expect(expectations.failed_expectations_data()).to_be_empty(); end; + procedure unord_incl_cols_as_list + as + l_actual sys_refcursor; + l_expected sys_refcursor; + begin + --Arrange + open l_actual for select rownum as rn, 'a' as "A_Column", 'c' as A_COLUMN, 'x' SOME_COL, 'd' "Some_Col" from dual a connect by level < 4; + open l_expected for select rownum as rn, 'a' as "A_Column", 'd' as A_COLUMN, 'x' SOME_COL, 'c' "Some_Col" from dual a connect by level < 4; + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).include(ut3.ut_varchar2_list('RN','//A_Column','SOME_COL')).unordered; + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + end; + + procedure joinby_incl_cols_as_list + as + l_actual sys_refcursor; + l_expected sys_refcursor; + begin + --Arrange + open l_actual for select rownum as rn, 'a' as "A_Column", 'c' as A_COLUMN, 'x' SOME_COL, 'd' "Some_Col" from dual a connect by level < 4; + open l_expected for select rownum as rn, 'a' as "A_Column", 'd' as A_COLUMN, 'x' SOME_COL, 'c' "Some_Col" from dual a connect by level < 4; + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).include(ut3.ut_varchar2_list('RN','//A_Column','SOME_COL')).join_by('SOME_COL'); + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + end; + + procedure joinby_excl_cols_as_list + as + l_actual sys_refcursor; + l_expected sys_refcursor; + begin + --Arrange + open l_actual for select rownum as rn, 'a' as "A_Column", 'c' as A_COLUMN, 'x' SOME_COL, 'd' "Some_Col" from dual a connect by level < 4; + open l_expected for select rownum as rn, 'a' as "A_Column", 'd' as A_COLUMN, 'x' SOME_COL, 'c' "Some_Col" from dual a connect by level < 4; + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).exclude(ut3.ut_varchar2_list('//Some_Col','A_COLUMN')).join_by('SOME_COL'); + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + end; + + procedure unord_excl_cols_as_list + as + l_actual sys_refcursor; + l_expected sys_refcursor; + begin + --Arrange + open l_actual for select rownum as rn, 'a' as "A_Column", 'c' as A_COLUMN, 'x' SOME_COL, 'd' "Some_Col" from dual a connect by level < 4; + open l_expected for select rownum as rn, 'a' as "A_Column", 'd' as A_COLUMN, 'x' SOME_COL, 'c' "Some_Col" from dual a connect by level < 4; + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).exclude(ut3.ut_varchar2_list('A_COLUMN|//Some_Col')).unordered; + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + end; + + procedure excl_dif_cols_as_list + as + l_actual sys_refcursor; + l_expected sys_refcursor; + begin + --Arrange + open l_actual for select rownum as rn, 'TEST' as A_COLUMN from dual a connect by level < 4; + open l_expected for select rownum as rn, 1 as A_COLUMN from dual a connect by level < 4; + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).exclude(ut3.ut_varchar2_list('A_COLUMN')); + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + end; + + procedure inlc_dif_cols_as_list + as + l_actual sys_refcursor; + l_expected sys_refcursor; + begin + --Arrange + open l_actual for select rownum as rn, 'TEST' as A_COLUMN from dual a connect by level < 4; + open l_expected for select rownum as rn, 1 as A_COLUMN from dual a connect by level < 4; + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).include(ut3.ut_varchar2_list('RN')); + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + end; + + procedure inlc_exc_dif_cols_as_list + as + l_actual sys_refcursor; + l_expected sys_refcursor; + begin + --Arrange + open l_actual for select rownum as rn, 'TEST' as A_COLUMN from dual a connect by level < 4; + open l_expected for select rownum as rn, 1 as A_COLUMN from dual a connect by level < 4; + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).include(ut3.ut_varchar2_list('RN')).exclude(ut3.ut_varchar2_list('A_COLUMN')); + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + end; + end; / diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pks b/test/core/expectations/compound_data/test_expectations_cursor.pks index 3447b4a15..76dc1fba8 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pks +++ b/test/core/expectations/compound_data/test_expectations_cursor.pks @@ -214,5 +214,26 @@ create or replace package test_expectations_cursor is --%test( Compare cursors join by single key more than 1000 rows) procedure cursor_joinby_compare_1000; + --%test(Unordered List of columns to include) + procedure unord_incl_cols_as_list; + + --%test(Join By List of columns to include) + procedure joinby_incl_cols_as_list; + + --%test(Unordered List of columns to exclude) + procedure unord_excl_cols_as_list; + + --%test(Join By List of columns to exclude) + procedure joinby_excl_cols_as_list; + + --%test(Exclude columns of diffrent type) + procedure excl_dif_cols_as_list; + + --%test(Include column of same type leaving diffrent type out) + procedure inlc_dif_cols_as_list; + + --%test(Include column of same type leaving diffrent type out and exclude diffrent type) + procedure inlc_exc_dif_cols_as_list; + end; / From ac042d05403f6301dc14f908b253c70b97b6268f Mon Sep 17 00:00:00 2001 From: lwasylow Date: Thu, 10 May 2018 18:00:23 +0100 Subject: [PATCH 12/24] Added failure on missing join by key in cursor. --- .../data_values/ut_compound_data_helper.pkb | 42 +++++++++++++++++ .../data_values/ut_compound_data_helper.pks | 10 ++++ .../data_values/ut_data_value_refcursor.tpb | 47 +++++++++++++++---- 3 files changed, 91 insertions(+), 8 deletions(-) diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index 85bc9fe2b..e1e9c936c 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -464,6 +464,48 @@ create or replace package body ut_compound_data_helper is return l_cols_hash; end; + function is_pk_exists(a_expected_cursor xmltype,a_actual_cursor xmltype, a_exclude_xpath varchar2, a_include_xpath varchar2,a_join_by_xpath varchar2) + return tt_missing_pk is + l_pk_xpath_tabs ut_varchar2_list := ut_varchar2_list(); + l_column_filter varchar2(32767); + l_no_missing_keys tt_missing_pk := tt_missing_pk(); + begin + if a_join_by_xpath is not null then + l_pk_xpath_tabs := ut_utils.string_to_table(a_join_by_xpath,'|'); + l_column_filter := get_columns_row_filter(a_exclude_xpath, a_include_xpath); + + execute immediate q'[ + with xpaths_tab as (select column_value xpath from table(:xpath_tabs)), + expected_column_info as ( select :expected as item_data from dual ), + actual_column_info as ( select :actual as item_data from dual ) + select xpath,diif_type from + ( + (select xpath,'e' diif_type from xpaths_tab + minus + select xpath,'e' diif_type + from ( select ]'||l_column_filter||q'[ from expected_column_info ucd) x + ,xpaths_tab + where xmlexists (xpaths_tab.xpath passing x.item_data) + ) + union all + (select xpath,'a' diif_type from xpaths_tab + minus + select xpath,'a' diif_type + from ( select ]'||l_column_filter||q'[ from actual_column_info ucd) x + ,xpaths_tab + where xmlexists (xpaths_tab.xpath passing x.item_data) + ) + )]' bulk collect into l_no_missing_keys + using l_pk_xpath_tabs,a_expected_cursor,a_actual_cursor, + a_exclude_xpath, a_include_xpath, + a_exclude_xpath, a_include_xpath; + + end if; + + return l_no_missing_keys; + end; + + begin g_type_name_map( dbms_sql.binary_bouble_type ) := 'BINARY_DOUBLE'; g_type_name_map( dbms_sql.bfile_type ) := 'BFILE'; diff --git a/source/expectations/data_values/ut_compound_data_helper.pks b/source/expectations/data_values/ut_compound_data_helper.pks index d71af802d..99aa7c18a 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pks +++ b/source/expectations/data_values/ut_compound_data_helper.pks @@ -28,6 +28,13 @@ create or replace package ut_compound_data_helper authid definer is type tt_column_diffs is table of t_column_diffs; + type t_missing_pk is record( + missingxpath varchar2(250), + diff_type varchar2(1) + ); + + type tt_missing_pk is table of t_missing_pk; + type t_row_diffs is record( rn integer, diff_type varchar2(250), @@ -72,6 +79,9 @@ create or replace package ut_compound_data_helper authid definer is a_data_value_cursor ut_data_value_refcursor, a_exclude_xpath varchar2, a_include_xpath varchar2, a_hash_type binary_integer := dbms_crypto.hash_sh1 ) return t_hash; + + function is_pk_exists(a_expected_cursor xmltype, a_actual_cursor xmltype, a_exclude_xpath varchar2, a_include_xpath varchar2,a_join_by_xpath varchar2) + return tt_missing_pk; end; / diff --git a/source/expectations/data_values/ut_data_value_refcursor.tpb b/source/expectations/data_values/ut_data_value_refcursor.tpb index 480aba30c..84746fb51 100644 --- a/source/expectations/data_values/ut_data_value_refcursor.tpb +++ b/source/expectations/data_values/ut_data_value_refcursor.tpb @@ -115,7 +115,8 @@ create or replace type body ut_data_value_refcursor as l_actual ut_data_value_refcursor; l_column_diffs ut_compound_data_helper.tt_column_diffs := ut_compound_data_helper.tt_column_diffs(); l_exclude_xpath varchar2(32767) := a_exclude_xpath; - + l_missing_pk ut_compound_data_helper.tt_missing_pk := ut_compound_data_helper.tt_missing_pk(); + function get_col_diff_text(a_col ut_compound_data_helper.t_column_diffs) return varchar2 is begin return @@ -130,7 +131,18 @@ create or replace type body ut_data_value_refcursor as ' Column <'||a_col.actual_name||'> is misplaced. Expected position: '||a_col.expected_pos||',' ||' actual position: '||a_col.actual_pos||'.' end; end; - + + function get_missing_key_message(a_missing_keys ut_compound_data_helper.t_missing_pk) return varchar2 is + begin + return + case a_missing_keys.diff_type + when 'a' then + ' Unknown key to join by in actual:'||a_missing_keys.missingxpath + when 'e' then + ' Unknown key to join by in expected:'||a_missing_keys.missingxpath + end; + end; + function add_incomparable_cols_to_xpath( a_column_diffs ut_compound_data_helper.tt_column_diffs, a_exclude_xpath varchar2 ) return varchar2 is @@ -151,6 +163,7 @@ create or replace type body ut_data_value_refcursor as end if; return l_result; end; + begin if not a_other is of (ut_data_value_refcursor) then raise value_error; @@ -176,12 +189,20 @@ create or replace type body ut_data_value_refcursor as end if; --diff rows and row elements - if a_join_by_xpath is not null then - ut_utils.append_to_clob(l_result, self.get_data_diff(a_other, a_exclude_xpath, a_include_xpath, a_join_by_xpath)); - elsif a_unordered then - ut_utils.append_to_clob(l_result, self.get_data_diff(a_other, l_exclude_xpath, a_include_xpath, a_unordered)); + if (a_join_by_xpath is not null) or not(a_unordered) then + -- Check if pk filter exists + l_missing_pk := ut_compound_data_helper.is_pk_exists(self.columns_info, l_actual.columns_info, a_exclude_xpath, a_include_xpath,a_join_by_xpath); + if l_missing_pk.count = 0 then + ut_utils.append_to_clob(l_result, self.get_data_diff(a_other, a_exclude_xpath, a_include_xpath, a_join_by_xpath)); + else + ut_utils.append_to_clob(l_result,chr(10) || 'Unable to join sets:' || chr(10)); + for i in 1 .. l_missing_pk.count loop + l_results.extend; + ut_utils.append_to_clob(l_result, get_missing_key_message(l_missing_pk(i))); + end loop; + end if; else - ut_utils.append_to_clob(l_result, self.get_data_diff(a_other, l_exclude_xpath, a_include_xpath, a_join_by_xpath)); + ut_utils.append_to_clob(l_result, self.get_data_diff(a_other, l_exclude_xpath, a_include_xpath, a_unordered)); end if; l_result_string := ut_utils.to_string(l_result,null); @@ -211,19 +232,29 @@ create or replace type body ut_data_value_refcursor as overriding member function compare_implementation (a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean) return integer is l_result integer := 0; l_other ut_data_value_refcursor; + l_missing_pk ut_compound_data_helper.tt_missing_pk := ut_compound_data_helper.tt_missing_pk(); begin if not a_other is of (ut_data_value_refcursor) then raise value_error; end if; l_other := treat(a_other as ut_data_value_refcursor); + + l_missing_pk := ut_compound_data_helper.is_pk_exists(self.columns_info, l_other.columns_info, a_exclude_xpath, a_include_xpath,a_join_by_xpath); + --if we join by key and key is missing fail and report error + if a_join_by_xpath is not null and l_missing_pk.count > 0 then + l_result := 1; + return l_result; + end if; + --if column names/types are not equal - build a diff of column names and types if ut_compound_data_helper.columns_hash( self, a_exclude_xpath, a_include_xpath ) != ut_compound_data_helper.columns_hash( l_other, a_exclude_xpath, a_include_xpath ) then l_result := 1; end if; - l_result := l_result + (self as ut_compound_data_value).compare_implementation(a_other, a_exclude_xpath, a_include_xpath, a_join_by_xpath, a_unordered); + + l_result := l_result + (self as ut_compound_data_value).compare_implementation(a_other, a_exclude_xpath, a_include_xpath, a_join_by_xpath, a_unordered); return l_result; end; From 38aba4389ec6ed5d17dee930bb0abbb19dca59a5 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Thu, 10 May 2018 22:15:34 +0100 Subject: [PATCH 13/24] Added unique key name as part of compare results Small fixes and optimizations --- .../data_values/ut_compound_data_diff_tmp.sql | 1 - .../data_values/ut_compound_data_helper.pkb | 54 +++++++++++++------ .../data_values/ut_compound_data_helper.pks | 2 + .../data_values/ut_compound_data_value.tpb | 31 +++-------- 4 files changed, 47 insertions(+), 41 deletions(-) diff --git a/source/expectations/data_values/ut_compound_data_diff_tmp.sql b/source/expectations/data_values/ut_compound_data_diff_tmp.sql index 95cb2f1f0..27f95907b 100644 --- a/source/expectations/data_values/ut_compound_data_diff_tmp.sql +++ b/source/expectations/data_values/ut_compound_data_diff_tmp.sql @@ -15,7 +15,6 @@ create global temporary table ut_compound_data_diff_tmp( diff_id raw(128), item_no integer, pk_hash raw(128), - pk_value varchar2(4000), item_hash raw(128), duplicate_no integer, constraint ut_compound_data_diff_tmp_uk1 unique (diff_id,duplicate_no,item_no,item_hash, pk_hash), diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index e1e9c936c..1cf111f80 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -186,7 +186,19 @@ create or replace package body ut_compound_data_helper is return l_results; end; - + + function get_pk_value (a_join_by_xpath varchar2,a_item_data xmltype) return varchar2 is + l_pk_value varchar2(4000); + begin + select listagg(extractvalue(xmlelement("ROW",column_value),a_join_by_xpath),':') within group ( order by 1) + into l_pk_value + from table(xmlsequence(extract(a_item_data,'/*/*'))); + + return l_pk_value; + exception when no_data_found then + return 'null '; + end; + function get_rows_diff( a_expected_dataset_guid raw, a_actual_dataset_guid raw, a_diff_id raw, a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2, @@ -195,15 +207,15 @@ create or replace package body ut_compound_data_helper is l_column_filter varchar2(32767); l_results tt_row_diffs; begin - l_column_filter := get_columns_filter(a_exclude_xpath,a_include_xpath); + l_column_filter := get_columns_row_filter(a_exclude_xpath,a_include_xpath); /** * Since its unordered search we cannot select max rows from diffs as we miss some comparision records * We will restrict output on higher level of select **/ - + execute immediate q'[ - with diff_info as (select item_hash,pk_hash,pk_value from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) + with diff_info as (select item_hash,pk_hash,pk_value, duplicate_no from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) select rn,diff_type,diffed_row,pk_value from ( select diff_type,diffed_row, dense_rank() over (order by pk_hash) rn,pk_value from @@ -215,16 +227,17 @@ create or replace package body ut_compound_data_helper is xmlserialize(content exp.row_data no indent) exp_item, xmlserialize(content act.row_data no indent) act_item from - (select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no + (select ucd.* from (select ucd.column_value row_data, r.item_hash row_hash, r.pk_hash , - r.pk_value, + r.duplicate_no, + ucd.column_value.getclobval() col_val, ucd.column_value.getRootElement() col_name, - ucd.column_value.getclobval() col_val + ut_compound_data_helper.get_pk_value(:join_xpath,r.item_data) pk_value from - (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash, i.pk_value + (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash, i.duplicate_no from ut_compound_data_tmp ucd, diff_info i where ucd.data_id = :self_guid @@ -234,16 +247,17 @@ create or replace package body ut_compound_data_helper is ) ucd ) exp join ( - select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no + select ucd.* from (select ucd.column_value row_data, r.item_hash row_hash, r.pk_hash , - r.pk_value, + r.duplicate_no, + ucd.column_value.getclobval() col_val, ucd.column_value.getRootElement() col_name, - ucd.column_value.getclobval() col_val + ut_compound_data_helper.get_pk_value(:join_xpath,r.item_data) pk_value from - (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash, i.pk_value + (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash, i.duplicate_no from ut_compound_data_tmp ucd, diff_info i where ucd.data_id = :other_guid @@ -252,7 +266,8 @@ create or replace package body ut_compound_data_helper is table( xmlsequence( extract(r.item_data,'/*/*') ) ) ucd ) ucd ) act - on exp.pk_hash = act.pk_hash and exp.duplicate_no = act.duplicate_no + on exp.pk_hash = act.pk_hash and exp.col_name = act.col_name + and exp.duplicate_no = act.duplicate_no where dbms_lob.compare(exp.col_val, act.col_val) != 0 ) unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:') ) @@ -261,15 +276,17 @@ create or replace package body ut_compound_data_helper is select case when exp.pk_hash is null then 'Extra:' else 'Missing:' end as diff_type, xmlserialize(content nvl(exp.item_data, act.item_data) no indent) diffed_row, coalesce(exp.pk_hash,act.pk_hash) pk_hash, - null pk_value - from (select extract(ucd.item_data,'/*/*') item_data,i.pk_hash + coalesce(exp.pk_value,act.pk_value) pk_value + from (select extract(ucd.item_data,'/*/*') item_data,i.pk_hash, + ut_compound_data_helper.get_pk_value(:join_by,item_data) pk_value from ut_compound_data_tmp ucd, diff_info i where ucd.data_id = :self_guid and ucd.item_hash = i.item_hash ) exp full outer join ( - select extract(ucd.item_data,'/*/*') item_data,i.pk_hash + select extract(ucd.item_data,'/*/*') item_data,i.pk_hash, + ut_compound_data_helper.get_pk_value(:join_by,item_data) pk_value from ut_compound_data_tmp ucd, diff_info i where ucd.data_id = :other_guid @@ -282,10 +299,13 @@ create or replace package body ut_compound_data_helper is ]' bulk collect into l_results using a_diff_id, + a_join_by_xpath, a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, + a_join_by_xpath, a_exclude_xpath, a_include_xpath, a_actual_dataset_guid, - a_expected_dataset_guid, a_actual_dataset_guid, + a_join_by_xpath,a_expected_dataset_guid,a_join_by_xpath, a_actual_dataset_guid, a_max_rows; + return l_results; end; diff --git a/source/expectations/data_values/ut_compound_data_helper.pks b/source/expectations/data_values/ut_compound_data_helper.pks index 99aa7c18a..a73f4c26f 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pks +++ b/source/expectations/data_values/ut_compound_data_helper.pks @@ -60,6 +60,8 @@ create or replace package ut_compound_data_helper authid definer is a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2 ) return tt_row_diffs; + function get_pk_value (a_join_by_xpath varchar2,a_item_data xmltype) return varchar2; + function get_rows_diff( a_expected_dataset_guid raw, a_actual_dataset_guid raw, a_diff_id raw, a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2, diff --git a/source/expectations/data_values/ut_compound_data_value.tpb b/source/expectations/data_values/ut_compound_data_value.tpb index efddc1c0b..889db7ee8 100644 --- a/source/expectations/data_values/ut_compound_data_value.tpb +++ b/source/expectations/data_values/ut_compound_data_value.tpb @@ -251,21 +251,6 @@ create or replace type body ut_compound_data_value as l_row_diffs ut_compound_data_helper.tt_row_diffs; c_max_rows constant integer := 20; - function get_pk_value(a_join_by_xpath varchar2) return varchar2 is - l_column varchar2(32767); - begin - /* due to possibility of key being to columns we cannot use xmlextractvalue - usage of xmlagg is possible however it greatly complicates code and performance is impacted. - xpath to be looked at or regex - */ - if a_join_by_xpath is not null then - l_column := ' extract(t.item_data,:join_by_xpath).GetStringVal() pk_value '; - else - l_column := ' :join_by_xpath pk_value'; - end if; - return l_column; - end; - function get_column_pk_hash(a_join_by_xpath varchar2) return varchar2 is l_column varchar2(32767); begin @@ -313,35 +298,35 @@ create or replace type body ut_compound_data_value as using a_exclude_xpath, a_include_xpath,a_join_by_xpath,self.data_id, l_other.data_id; /* Peform minus on two sets two get diffrences that will be used later on to print results */ - execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no,pk_value) + execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no) with source_data as ( select t.data_id,t.item_hash,row_number() over (partition by t.pk_hash,t.item_hash,t.data_id order by 1,2) duplicate_no, - pk_hash, '||get_pk_value(a_join_by_xpath)||' + pk_hash from ' || l_ut_owner || '.ut_compound_data_tmp t where data_id = :self_guid or data_id = :other_guid ) - select distinct :diff_id,tmp.item_hash,tmp.pk_hash,tmp.duplicate_no,pk_value + select distinct :diff_id,tmp.item_hash,tmp.pk_hash,tmp.duplicate_no from( ( - select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value + select t.item_hash,t. duplicate_no,t.pk_hash from source_data t where t.data_id = :self_guid minus - select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value + select t.item_hash,t. duplicate_no,t.pk_hash from source_data t where t.data_id = :other_guid ) union all ( - select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value + select t.item_hash,t. duplicate_no,t.pk_hash from source_data t where t.data_id = :other_guid minus - select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value + select t.item_hash,t. duplicate_no,t.pk_hash from source_data t where t.data_id = :self_guid ))tmp' - using a_join_by_xpath,self.data_id, l_other.data_id, + using self.data_id, l_other.data_id, l_diff_id, self.data_id, l_other.data_id, l_other.data_id,self.data_id; From 66974e000d77fc2575866109bc6e92adb24ba238 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Thu, 10 May 2018 22:15:34 +0100 Subject: [PATCH 14/24] Added unique key name as part of compare results Small fixes and optimizations --- .../data_values/ut_compound_data_diff_tmp.sql | 1 - .../data_values/ut_compound_data_helper.pkb | 57 +++-- .../data_values/ut_compound_data_helper.pks | 2 + .../data_values/ut_compound_data_value.tpb | 31 +-- .../data_values/ut_data_value_refcursor.tpb | 2 +- .../test_expectations_cursor.pkb | 216 ++++++++++++++++++ .../test_expectations_cursor.pks | 35 ++- 7 files changed, 297 insertions(+), 47 deletions(-) diff --git a/source/expectations/data_values/ut_compound_data_diff_tmp.sql b/source/expectations/data_values/ut_compound_data_diff_tmp.sql index 95cb2f1f0..27f95907b 100644 --- a/source/expectations/data_values/ut_compound_data_diff_tmp.sql +++ b/source/expectations/data_values/ut_compound_data_diff_tmp.sql @@ -15,7 +15,6 @@ create global temporary table ut_compound_data_diff_tmp( diff_id raw(128), item_no integer, pk_hash raw(128), - pk_value varchar2(4000), item_hash raw(128), duplicate_no integer, constraint ut_compound_data_diff_tmp_uk1 unique (diff_id,duplicate_no,item_no,item_hash, pk_hash), diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index e1e9c936c..883f9a364 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -186,7 +186,19 @@ create or replace package body ut_compound_data_helper is return l_results; end; - + + function get_pk_value (a_join_by_xpath varchar2,a_item_data xmltype) return varchar2 is + l_pk_value varchar2(4000); + begin + select listagg(extractvalue(xmlelement("ROW",column_value),a_join_by_xpath),':') within group ( order by 1) + into l_pk_value + from table(xmlsequence(extract(a_item_data,'/*/*'))); + + return l_pk_value; + exception when no_data_found then + return 'null '; + end; + function get_rows_diff( a_expected_dataset_guid raw, a_actual_dataset_guid raw, a_diff_id raw, a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2, @@ -195,15 +207,16 @@ create or replace package body ut_compound_data_helper is l_column_filter varchar2(32767); l_results tt_row_diffs; begin - l_column_filter := get_columns_filter(a_exclude_xpath,a_include_xpath); + l_column_filter := get_columns_row_filter(a_exclude_xpath,a_include_xpath); /** * Since its unordered search we cannot select max rows from diffs as we miss some comparision records * We will restrict output on higher level of select + * NO_MERGE hint was introduced to prevent optimizer from merging views and rewriting query which in some cases + * lead to second value being null depend on execution plan that been chosen **/ - execute immediate q'[ - with diff_info as (select item_hash,pk_hash,pk_value from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) + with diff_info as (select item_hash,pk_hash,duplicate_no from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) select rn,diff_type,diffed_row,pk_value from ( select diff_type,diffed_row, dense_rank() over (order by pk_hash) rn,pk_value from @@ -211,20 +224,21 @@ create or replace package body ut_compound_data_helper is select diff_type,diffed_row,pk_hash,pk_value from (select diff_type,data_item diffed_row,pk_hash,pk_value from - (select nvl(exp.pk_hash, act.pk_hash) pk_hash,nvl(exp.pk_value, act.pk_value) pk_value, + (select /*+NO_MERGE*/ nvl(exp.pk_hash, act.pk_hash) pk_hash,nvl(exp.pk_value, act.pk_value) pk_value, xmlserialize(content exp.row_data no indent) exp_item, xmlserialize(content act.row_data no indent) act_item from - (select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no + (select ucd.* from (select ucd.column_value row_data, r.item_hash row_hash, r.pk_hash , - r.pk_value, + r.duplicate_no, + ucd.column_value.getclobval() col_val, ucd.column_value.getRootElement() col_name, - ucd.column_value.getclobval() col_val + ut_compound_data_helper.get_pk_value(:join_xpath,r.item_data) pk_value from - (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash, i.pk_value + (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash, i.duplicate_no from ut_compound_data_tmp ucd, diff_info i where ucd.data_id = :self_guid @@ -234,16 +248,17 @@ create or replace package body ut_compound_data_helper is ) ucd ) exp join ( - select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no + select ucd.* from (select ucd.column_value row_data, r.item_hash row_hash, r.pk_hash , - r.pk_value, + r.duplicate_no, + ucd.column_value.getclobval() col_val, ucd.column_value.getRootElement() col_name, - ucd.column_value.getclobval() col_val + ut_compound_data_helper.get_pk_value(:join_xpath,r.item_data) pk_value from - (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash, i.pk_value + (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash, i.duplicate_no from ut_compound_data_tmp ucd, diff_info i where ucd.data_id = :other_guid @@ -252,7 +267,8 @@ create or replace package body ut_compound_data_helper is table( xmlsequence( extract(r.item_data,'/*/*') ) ) ucd ) ucd ) act - on exp.pk_hash = act.pk_hash and exp.duplicate_no = act.duplicate_no + on exp.pk_hash = act.pk_hash and exp.col_name = act.col_name + and exp.duplicate_no = act.duplicate_no where dbms_lob.compare(exp.col_val, act.col_val) != 0 ) unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:') ) @@ -261,15 +277,17 @@ create or replace package body ut_compound_data_helper is select case when exp.pk_hash is null then 'Extra:' else 'Missing:' end as diff_type, xmlserialize(content nvl(exp.item_data, act.item_data) no indent) diffed_row, coalesce(exp.pk_hash,act.pk_hash) pk_hash, - null pk_value - from (select extract(ucd.item_data,'/*/*') item_data,i.pk_hash + coalesce(exp.pk_value,act.pk_value) pk_value + from (select extract(ucd.item_data,'/*/*') item_data,i.pk_hash, + ut_compound_data_helper.get_pk_value(:join_by,item_data) pk_value from ut_compound_data_tmp ucd, diff_info i where ucd.data_id = :self_guid and ucd.item_hash = i.item_hash ) exp full outer join ( - select extract(ucd.item_data,'/*/*') item_data,i.pk_hash + select extract(ucd.item_data,'/*/*') item_data,i.pk_hash, + ut_compound_data_helper.get_pk_value(:join_by,item_data) pk_value from ut_compound_data_tmp ucd, diff_info i where ucd.data_id = :other_guid @@ -282,10 +300,13 @@ create or replace package body ut_compound_data_helper is ]' bulk collect into l_results using a_diff_id, + a_join_by_xpath, a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, + a_join_by_xpath, a_exclude_xpath, a_include_xpath, a_actual_dataset_guid, - a_expected_dataset_guid, a_actual_dataset_guid, + a_join_by_xpath,a_expected_dataset_guid,a_join_by_xpath, a_actual_dataset_guid, a_max_rows; + return l_results; end; diff --git a/source/expectations/data_values/ut_compound_data_helper.pks b/source/expectations/data_values/ut_compound_data_helper.pks index 99aa7c18a..a73f4c26f 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pks +++ b/source/expectations/data_values/ut_compound_data_helper.pks @@ -60,6 +60,8 @@ create or replace package ut_compound_data_helper authid definer is a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2 ) return tt_row_diffs; + function get_pk_value (a_join_by_xpath varchar2,a_item_data xmltype) return varchar2; + function get_rows_diff( a_expected_dataset_guid raw, a_actual_dataset_guid raw, a_diff_id raw, a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2, diff --git a/source/expectations/data_values/ut_compound_data_value.tpb b/source/expectations/data_values/ut_compound_data_value.tpb index efddc1c0b..889db7ee8 100644 --- a/source/expectations/data_values/ut_compound_data_value.tpb +++ b/source/expectations/data_values/ut_compound_data_value.tpb @@ -251,21 +251,6 @@ create or replace type body ut_compound_data_value as l_row_diffs ut_compound_data_helper.tt_row_diffs; c_max_rows constant integer := 20; - function get_pk_value(a_join_by_xpath varchar2) return varchar2 is - l_column varchar2(32767); - begin - /* due to possibility of key being to columns we cannot use xmlextractvalue - usage of xmlagg is possible however it greatly complicates code and performance is impacted. - xpath to be looked at or regex - */ - if a_join_by_xpath is not null then - l_column := ' extract(t.item_data,:join_by_xpath).GetStringVal() pk_value '; - else - l_column := ' :join_by_xpath pk_value'; - end if; - return l_column; - end; - function get_column_pk_hash(a_join_by_xpath varchar2) return varchar2 is l_column varchar2(32767); begin @@ -313,35 +298,35 @@ create or replace type body ut_compound_data_value as using a_exclude_xpath, a_include_xpath,a_join_by_xpath,self.data_id, l_other.data_id; /* Peform minus on two sets two get diffrences that will be used later on to print results */ - execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no,pk_value) + execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no) with source_data as ( select t.data_id,t.item_hash,row_number() over (partition by t.pk_hash,t.item_hash,t.data_id order by 1,2) duplicate_no, - pk_hash, '||get_pk_value(a_join_by_xpath)||' + pk_hash from ' || l_ut_owner || '.ut_compound_data_tmp t where data_id = :self_guid or data_id = :other_guid ) - select distinct :diff_id,tmp.item_hash,tmp.pk_hash,tmp.duplicate_no,pk_value + select distinct :diff_id,tmp.item_hash,tmp.pk_hash,tmp.duplicate_no from( ( - select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value + select t.item_hash,t. duplicate_no,t.pk_hash from source_data t where t.data_id = :self_guid minus - select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value + select t.item_hash,t. duplicate_no,t.pk_hash from source_data t where t.data_id = :other_guid ) union all ( - select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value + select t.item_hash,t. duplicate_no,t.pk_hash from source_data t where t.data_id = :other_guid minus - select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value + select t.item_hash,t. duplicate_no,t.pk_hash from source_data t where t.data_id = :self_guid ))tmp' - using a_join_by_xpath,self.data_id, l_other.data_id, + using self.data_id, l_other.data_id, l_diff_id, self.data_id, l_other.data_id, l_other.data_id,self.data_id; diff --git a/source/expectations/data_values/ut_data_value_refcursor.tpb b/source/expectations/data_values/ut_data_value_refcursor.tpb index 84746fb51..021b493ca 100644 --- a/source/expectations/data_values/ut_data_value_refcursor.tpb +++ b/source/expectations/data_values/ut_data_value_refcursor.tpb @@ -198,7 +198,7 @@ create or replace type body ut_data_value_refcursor as ut_utils.append_to_clob(l_result,chr(10) || 'Unable to join sets:' || chr(10)); for i in 1 .. l_missing_pk.count loop l_results.extend; - ut_utils.append_to_clob(l_result, get_missing_key_message(l_missing_pk(i))); + ut_utils.append_to_clob(l_result, get_missing_key_message(l_missing_pk(i))|| chr(10)); end loop; end if; else diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pkb b/test/core/expectations/compound_data/test_expectations_cursor.pkb index 5a0f429cc..12602c61f 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pkb +++ b/test/core/expectations/compound_data/test_expectations_cursor.pkb @@ -1114,6 +1114,144 @@ Extra: test-666%]'; ut.expect(expectations.failed_expectations_data()).to_be_empty(); end; + procedure cursor_joinby_compare_nokey is + l_actual SYS_REFCURSOR; + l_expected SYS_REFCURSOR; + l_actual_message varchar2(32767); + l_expected_message varchar2(32767); + begin + --Arrange + open l_actual for select rownum as rn, 'x' SOME_COL from dual a connect by level < 4; + open l_expected for select rownum as rn, 'x' SOME_COL from dual a connect by level < 4; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by('OWNER'); + --Assert + l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% +Diff:% +%Unable to join sets:% +%Unknown key to join by in expected:/*/OWNER% +%Unknown key to join by in actual:/*/OWNER%]'; + l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; + --Assert + ut.expect(l_actual_message).to_be_like(l_expected_message); + end; + + procedure cur_joinby_comp_twocols_nokey is + l_actual SYS_REFCURSOR; + l_expected SYS_REFCURSOR; + l_actual_message varchar2(32767); + l_expected_message varchar2(32767); + begin + --Arrange + open l_actual for select rownum as rn, 'x' SOME_COL from dual a connect by level < 4; + open l_expected for select rownum as rn, 'x' SOME_COL from dual a connect by level < 4; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by(ut3.ut_varchar2_list('OWNER,USER_ID')); + --Assert + l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% +Diff:% +%Unable to join sets:% +%Unknown key to join by in expected:/*/OWNER% +%Unknown key to join by in expected:/*/USER_ID% +%Unknown key to join by in actual:/*/OWNER% +%Unknown key to join by in actual:/*/USER_ID%]'; + l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; + --Assert + ut.expect(l_actual_message).to_be_like(l_expected_message); + end; + + procedure cursor_joinby_compare_exkey is + l_actual SYS_REFCURSOR; + l_expected SYS_REFCURSOR; + l_actual_message varchar2(32767); + l_expected_message varchar2(32767); + begin + --Arrange + open l_actual for select rownum as rn, 'x' SOME_COL from dual a connect by level < 4; + open l_expected for select rownum as rn, 'x' SOME_COL from dual a connect by level < 4; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by('SOME_COL').exclude('SOME_COL'); + --Assert + l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% +Diff:% +%Unable to join sets:% +%Unknown key to join by in expected:/*/SOME_COL% +%Unknown key to join by in actual:/*/SOME_COL%]'; + l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; + --Assert + ut.expect(l_actual_message).to_be_like(l_expected_message); + end; + + procedure cur_joinby_comp_twocols_exkey is + l_actual SYS_REFCURSOR; + l_expected SYS_REFCURSOR; + l_actual_message varchar2(32767); + l_expected_message varchar2(32767); + begin + --Arrange + open l_actual for select rownum as rn, 'x' SOME_COL from dual a connect by level < 4; + open l_expected for select rownum as rn, 'x' SOME_COL from dual a connect by level < 4; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by(ut3.ut_varchar2_list('RN,SOME_COL')).exclude('RN'); + --Assert + l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% +Diff:% +%Unable to join sets:% +%Unknown key to join by in expected:/*/RN% +%Unknown key to join by in actual:/*/RN%]'; + l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; + --Assert + ut.expect(l_actual_message).to_be_like(l_expected_message); + end; + + procedure cursor_joinby_comp_nokey_ex is + l_actual SYS_REFCURSOR; + l_expected SYS_REFCURSOR; + l_actual_message varchar2(32767); + l_expected_message varchar2(32767); + begin + --Arrange + open l_actual for select rownum as rni, 'x' SOME_COL from dual a connect by level < 4; + open l_expected for select rownum as rn, 'x' SOME_COL from dual a connect by level < 4; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by('RNI'); + --Assert + l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% +Diff:% +%Unable to join sets:% +%Unknown key to join by in expected:/*/RNI%]'; + l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; + --Assert + ut.expect(l_actual_message).to_be_like(l_expected_message); + end; + + procedure cursor_joinby_comp_nokey_ac is + l_actual SYS_REFCURSOR; + l_expected SYS_REFCURSOR; + l_actual_message varchar2(32767); + l_expected_message varchar2(32767); + begin + --Arrange + open l_actual for select rownum as rn, 'x' SOME_COL from dual a connect by level < 4; + open l_expected for select rownum as rni, 'x' SOME_COL from dual a connect by level < 4; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by('RNI'); + --Assert + l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% +Diff:% +%Unable to join sets:% +%Unknown key to join by in actual:/*/RNI%]'; + l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; + --Assert + ut.expect(l_actual_message).to_be_like(l_expected_message); + end; + procedure cursor_joinby_compare_1000 is l_actual SYS_REFCURSOR; l_expected SYS_REFCURSOR; @@ -1127,6 +1265,84 @@ Extra: test-666%]'; ut.expect(expectations.failed_expectations_data()).to_be_empty(); end; + procedure cursor_joinby_compare_fail is + l_actual SYS_REFCURSOR; + l_expected SYS_REFCURSOR; + l_actual_message varchar2(32767); + l_expected_message varchar2(32767); + begin + --Arrange + open l_expected for select username, user_id from all_users union all + select 'TEST' username, -600 user_id from dual order by 1 desc; + + open l_actual for select username, user_id from all_users union all + select 'TEST' username, -610 user_id from dual order by 1 asc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by('USERNAME'); + --Assert + l_expected_message := q'[%Actual: refcursor [ count = % ] was expected to equal: refcursor [ count = % ] +%Diff:% +%Rows: [ 1 differences ]% +%Expected: -600 for key: TEST% +%Actual: -610 for key: TEST%]'; + l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; + --Assert + ut.expect(l_actual_message).to_be_like(l_expected_message); + end; + + procedure cursor_joinby_cmp_twocol_fail is + l_actual SYS_REFCURSOR; + l_expected SYS_REFCURSOR; + l_actual_message varchar2(32767); + l_expected_message varchar2(32767); + begin + --Arrange + open l_expected for select username, user_id from all_users union all + select 'TEST' username, -600 user_id from dual order by 1 desc; + + open l_actual for select username, user_id from all_users union all + select 'TEST' username, -610 user_id from dual order by 1 asc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by(ut3.ut_varchar2_list('USERNAME,USER_ID')); + --Assert + l_expected_message := q'[%Actual: refcursor [ count = % ] was expected to equal: refcursor [ count = % ] +%Diff:% +%Rows: [ 2 differences ]% +%Missing: TEST-600 for key: -600:TEST% +%Extra: TEST-610 for key: -610:TEST%]'; + l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; + --Assert + ut.expect(l_actual_message).to_be_like(l_expected_message); + end; + + procedure cur_joinby_cmp_threcol_fail is + l_actual SYS_REFCURSOR; + l_expected SYS_REFCURSOR; + l_actual_message varchar2(32767); + l_expected_message varchar2(32767); + begin + --Arrange + open l_expected for select username, user_id,'Y' is_valid from all_users union all + select 'TEST' username, -600 user_id,'Y' is_valid from dual order by 1 desc; + + open l_actual for select username, user_id,'Y' is_valid from all_users union all + select 'TEST' username, -610 user_id,'Y' is_valid from dual order by 1 asc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by(ut3.ut_varchar2_list('USERNAME,IS_VALID')); + --Assert + l_expected_message := q'[%Actual: refcursor [ count = % ] was expected to equal: refcursor [ count = % ] +%Diff:% +%Rows: [ 1 differences ]% +%Expected: -600 for key: TEST:Y% +%Actual: -610 for key: TEST:Y%]'; + l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; + --Assert + ut.expect(l_actual_message).to_be_like(l_expected_message); + end; + procedure unord_incl_cols_as_list as l_actual sys_refcursor; diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pks b/test/core/expectations/compound_data/test_expectations_cursor.pks index 76dc1fba8..eb5144c29 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pks +++ b/test/core/expectations/compound_data/test_expectations_cursor.pks @@ -211,9 +211,36 @@ create or replace package test_expectations_cursor is --%test( Compare cursors join by composite key) procedure cursor_joinby_compare_twocols; + --%test( Compare cursors join by single key - key doesnt exists ) + procedure cursor_joinby_compare_nokey; + + --%test( Compare cursors join by composite key - one part of key doesnt exists ) + procedure cur_joinby_comp_twocols_nokey; + + --%test( Compare cursors join by single key - key doesnt is excluded ) + procedure cursor_joinby_compare_exkey; + + --%test( Compare cursors join by composite key - one part of key is excluded exists ) + procedure cur_joinby_comp_twocols_exkey; + + --%test( Compare cursors join by single key - key doesnt exists in expected) + procedure cursor_joinby_comp_nokey_ex; + + --%test( Compare cursors join by single key - key doesnt exists in actual) + procedure cursor_joinby_comp_nokey_ac; + --%test( Compare cursors join by single key more than 1000 rows) procedure cursor_joinby_compare_1000; + --%test( Compare two column cursors join by and fail to match ) + procedure cursor_joinby_compare_fail; + + --%test( Compare two column cursors join by two columns and fail to match ) + procedure cursor_joinby_cmp_twocol_fail; + + --%test( Compare three column cursors join by two columns and fail to match ) + procedure cur_joinby_cmp_threcol_fail; + --%test(Unordered List of columns to include) procedure unord_incl_cols_as_list; @@ -226,14 +253,14 @@ create or replace package test_expectations_cursor is --%test(Join By List of columns to exclude) procedure joinby_excl_cols_as_list; - --%test(Exclude columns of diffrent type) + --%test(Exclude columns of different type) procedure excl_dif_cols_as_list; - --%test(Include column of same type leaving diffrent type out) + --%test(Include column of same type leaving different type out) procedure inlc_dif_cols_as_list; - --%test(Include column of same type leaving diffrent type out and exclude diffrent type) + --%test(Include column of same type leaving different type out and exclude different type) procedure inlc_exc_dif_cols_as_list; - + end; / From 0fd8be8c4d4020a67e488ee57eae74b672e2606c Mon Sep 17 00:00:00 2001 From: lwasylow Date: Fri, 11 May 2018 13:22:48 +0100 Subject: [PATCH 15/24] Revert "Initial check in of unordered comparision" This reverts commit b9ca972d659191845f8391a7d7ca316a1ce3e16a. for one file --- source/expectations/data_values/ut_data_value_anydata.tps | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/expectations/data_values/ut_data_value_anydata.tps b/source/expectations/data_values/ut_data_value_anydata.tps index 87829cfa1..91d66c7bd 100644 --- a/source/expectations/data_values/ut_data_value_anydata.tps +++ b/source/expectations/data_values/ut_data_value_anydata.tps @@ -1,4 +1,4 @@ -create or replace type ut_data_value_anydata force under ut_compound_data_value( +create or replace type ut_data_value_anydata under ut_compound_data_value( /* utPLSQL - Version 3 Copyright 2016 - 2017 utPLSQL Project From 1bf5bad62e7ccaeab8fa2c077454daf06f29b086 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Fri, 11 May 2018 17:23:27 +0100 Subject: [PATCH 16/24] Update blank line --- source/expectations/data_values/ut_compound_data_helper.pkb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index 883f9a364..2d83ac634 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -549,4 +549,4 @@ begin g_type_name_map( dbms_sql.urowid_type ) := 'UROWID'; end; -/ \ No newline at end of file +/ From cc718d9c74f4fd550601a1d2bfc65283a2d9ec35 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Thu, 17 May 2018 23:55:00 +0100 Subject: [PATCH 17/24] Added changes to cater for nested tables Updated documentation Shift packages around for current user id --- docs/userguide/advanced_data_comparison.md | 4 +- source/core/types/ut_key_anyval_pair.tps | 20 ++ source/core/types/ut_key_anyval_pairs.tps | 19 ++ source/core/types/ut_key_value_pair.tps | 2 +- .../core/types/ut_key_varcharvalue_pair.tps | 20 ++ source/core/types/ut_key_xmlvalue_pair.tps | 20 ++ .../data_values/ut_compound_data_helper.pkb | 91 ++------- .../data_values/ut_compound_data_helper.pks | 2 +- .../ut_curr_usr_compound_helper.pkb | 186 ++++++++++++++++++ .../ut_curr_usr_compound_helper.pks | 8 + .../data_values/ut_data_value_refcursor.tpb | 11 +- .../data_values/ut_data_value_refcursor.tps | 9 +- source/install.sql | 6 + source/uninstall.sql | 8 + .../test_expectations_cursor.pkb | 135 +++++++++++-- .../test_expectations_cursor.pks | 20 +- 16 files changed, 466 insertions(+), 95 deletions(-) create mode 100644 source/core/types/ut_key_anyval_pair.tps create mode 100644 source/core/types/ut_key_anyval_pairs.tps create mode 100644 source/core/types/ut_key_varcharvalue_pair.tps create mode 100644 source/core/types/ut_key_xmlvalue_pair.tps create mode 100644 source/expectations/data_values/ut_curr_usr_compound_helper.pkb create mode 100644 source/expectations/data_values/ut_curr_usr_compound_helper.pks diff --git a/docs/userguide/advanced_data_comparison.md b/docs/userguide/advanced_data_comparison.md index e003b0c47..1edc39b7c 100644 --- a/docs/userguide/advanced_data_comparison.md +++ b/docs/userguide/advanced_data_comparison.md @@ -139,7 +139,9 @@ Above test will result in two differences of one row extra and one row missing. ## Join By option -You can now join two cursors by defining a primary key or composite key that will be used to uniquely identify and compare rows. This option allows us to exactly show which rows are missing, extra and which are different without ordering clause. +You can now join two cursors by defining a primary key or composite key that will be used to uniquely identify and compare rows. This option allows us to exactly show which rows are missing, extra and which are different without ordering clause. In the situation where the join key is not unique, join will partition set over rows with a same key and join on row number as well as given join key. The extra rows or missing will be presented to user as well as not matching rows. + +Join by option can be used in conjunction with include or exclude options. However if any of the join keys is part of exclude set, comparison will fail and report to user that sets could not be joined on specific key (excluded). ```sql procedure join_by_username is diff --git a/source/core/types/ut_key_anyval_pair.tps b/source/core/types/ut_key_anyval_pair.tps new file mode 100644 index 000000000..5188bdda5 --- /dev/null +++ b/source/core/types/ut_key_anyval_pair.tps @@ -0,0 +1,20 @@ +create or replace type ut_key_anyval_pair force as object( + /* + utPLSQL - Version 3 + Copyright 2016 - 2017 utPLSQL Project + + Licensed under the Apache License, Version 2.0 (the "License"): + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + key varchar2(4000) +) not final +/ diff --git a/source/core/types/ut_key_anyval_pairs.tps b/source/core/types/ut_key_anyval_pairs.tps new file mode 100644 index 000000000..ccf4ce047 --- /dev/null +++ b/source/core/types/ut_key_anyval_pairs.tps @@ -0,0 +1,19 @@ +create or replace type ut_key_anyval_pairs as + /* + utPLSQL - Version 3 + Copyright 2016 - 2017 utPLSQL Project + + Licensed under the Apache License, Version 2.0 (the "License"): + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + table of ut_key_anyval_pair +/ diff --git a/source/core/types/ut_key_value_pair.tps b/source/core/types/ut_key_value_pair.tps index 45bc53157..a23df4a7b 100644 --- a/source/core/types/ut_key_value_pair.tps +++ b/source/core/types/ut_key_value_pair.tps @@ -1,4 +1,4 @@ -create or replace type ut_key_value_pair as object( +create or replace type ut_key_value_pair force as object( /* utPLSQL - Version 3 Copyright 2016 - 2017 utPLSQL Project diff --git a/source/core/types/ut_key_varcharvalue_pair.tps b/source/core/types/ut_key_varcharvalue_pair.tps new file mode 100644 index 000000000..fdce93cf0 --- /dev/null +++ b/source/core/types/ut_key_varcharvalue_pair.tps @@ -0,0 +1,20 @@ +create or replace type ut_key_varcharvalue_pair under ut_key_anyval_pair ( + /* + utPLSQL - Version 3 + Copyright 2016 - 2017 utPLSQL Project + + Licensed under the Apache License, Version 2.0 (the "License"): + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + value varchar2(4000) +) +/ diff --git a/source/core/types/ut_key_xmlvalue_pair.tps b/source/core/types/ut_key_xmlvalue_pair.tps new file mode 100644 index 000000000..1f5462861 --- /dev/null +++ b/source/core/types/ut_key_xmlvalue_pair.tps @@ -0,0 +1,20 @@ +create or replace type ut_key_xmlvalue_pair under ut_key_anyval_pair ( + /* + utPLSQL - Version 3 + Copyright 2016 - 2017 utPLSQL Project + + Licensed under the Apache License, Version 2.0 (the "License"): + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + value xmltype +) +/ diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index 3528b4ec8..71963263f 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -16,55 +16,23 @@ create or replace package body ut_compound_data_helper is limitations under the License. */ - type t_type_name_map is table of varchar2(100) index by binary_integer; - g_type_name_map t_type_name_map; - - function get_column_type(a_desc_rec dbms_sql.desc_rec3) return varchar2 is - l_result varchar2(500) := 'unknown datatype'; - begin - if g_type_name_map.exists(a_desc_rec.col_type) then - l_result := g_type_name_map(a_desc_rec.col_type); - elsif a_desc_rec.col_schema_name is not null and a_desc_rec.col_type_name is not null then - l_result := a_desc_rec.col_schema_name||'.'||a_desc_rec.col_type_name; - end if; - return l_result; - end; - - function get_columns_info(a_columns_tab dbms_sql.desc_tab3, a_columns_count integer) return ut_key_value_pairs is - l_result ut_key_value_pairs := ut_key_value_pairs(); - begin - for i in 1 .. a_columns_count loop - l_result.extend; - l_result(l_result.last) := ut_key_value_pair(a_columns_tab(i).col_name, get_column_type(a_columns_tab(i))); - end loop; - return l_result; - end; - - function get_columns_info(a_cursor in out nocopy sys_refcursor) return xmltype is - l_cursor_number integer; - l_columns_count pls_integer; - l_columns_desc dbms_sql.desc_tab3; - l_result xmltype; - l_columns_tab ut_key_value_pairs; - begin - if a_cursor is null or not a_cursor%isopen then - return null; - end if; - l_cursor_number := dbms_sql.to_cursor_number( a_cursor ); - dbms_sql.describe_columns3( l_cursor_number, l_columns_count, l_columns_desc ); - a_cursor := dbms_sql.to_refcursor( l_cursor_number ); - l_columns_tab := get_columns_info( l_columns_desc, l_columns_count); - - select - XMLELEMENT("ROW", xmlagg(xmlelement(evalname ut_utils.xmlgen_escaped_string(key), - XMLATTRIBUTES(key AS "xml_valid_name"), - value))) - into l_result - from table(l_columns_tab ); - - return l_result; - end; - + g_user_defined_type pls_integer := dbms_sql.user_defined_type; + + function get_column_info_xml(a_column_details ut_key_anyval_pair) return xmltype is + l_result varchar2(4000); + l_res xmltype; + begin + l_result := '<'||ut_utils.xmlgen_escaped_string(a_column_details.KEY)||' xml_valid_name="'||ut_utils.xmlgen_escaped_string(a_column_details.KEY)||'">'; + if a_column_details is of(ut_key_xmlvalue_pair) then + l_result := l_result || (treat(a_column_details as ut_key_xmlvalue_pair).value.getstringval()); + else + l_result := l_result || ut_utils.xmlgen_escaped_string((treat(a_column_details as ut_key_varcharvalue_pair).value)); + end if; + l_result := l_result ||''; + + return xmltype(l_result); + end; + function get_columns_filter( a_exclude_xpath varchar2, a_include_xpath varchar2, a_table_alias varchar2 := 'ucd', a_column_alias varchar2 := 'item_data' @@ -524,6 +492,7 @@ create or replace package body ut_compound_data_helper is l_pk_xpath_tabs ut_varchar2_list := ut_varchar2_list(); l_column_filter varchar2(32767); l_no_missing_keys tt_missing_pk := tt_missing_pk(); + begin if a_join_by_xpath is not null then l_pk_xpath_tabs := ut_utils.string_to_table(a_join_by_xpath,'|'); @@ -559,28 +528,6 @@ create or replace package body ut_compound_data_helper is return l_no_missing_keys; end; - - -begin - g_type_name_map( dbms_sql.binary_bouble_type ) := 'BINARY_DOUBLE'; - g_type_name_map( dbms_sql.bfile_type ) := 'BFILE'; - g_type_name_map( dbms_sql.binary_float_type ) := 'BINARY_FLOAT'; - g_type_name_map( dbms_sql.blob_type ) := 'BLOB'; - g_type_name_map( dbms_sql.long_raw_type ) := 'LONG RAW'; - g_type_name_map( dbms_sql.char_type ) := 'CHAR'; - g_type_name_map( dbms_sql.clob_type ) := 'CLOB'; - g_type_name_map( dbms_sql.long_type ) := 'LONG'; - g_type_name_map( dbms_sql.date_type ) := 'DATE'; - g_type_name_map( dbms_sql.interval_day_to_second_type ) := 'INTERVAL DAY TO SECOND'; - g_type_name_map( dbms_sql.interval_year_to_month_type ) := 'INTERVAL YEAR TO MONTH'; - g_type_name_map( dbms_sql.raw_type ) := 'RAW'; - g_type_name_map( dbms_sql.timestamp_type ) := 'TIMESTAMP'; - g_type_name_map( dbms_sql.timestamp_with_tz_type ) := 'TIMESTAMP WITH TIME ZONE'; - g_type_name_map( dbms_sql.timestamp_with_local_tz_type ) := 'TIMESTAMP WITH LOCAL TIME ZONE'; - g_type_name_map( dbms_sql.varchar2_type ) := 'VARCHAR2'; - g_type_name_map( dbms_sql.number_type ) := 'NUMBER'; - g_type_name_map( dbms_sql.rowid_type ) := 'ROWID'; - g_type_name_map( dbms_sql.urowid_type ) := 'UROWID'; - + end; / diff --git a/source/expectations/data_values/ut_compound_data_helper.pks b/source/expectations/data_values/ut_compound_data_helper.pks index e497d7223..6558182ae 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pks +++ b/source/expectations/data_values/ut_compound_data_helper.pks @@ -48,7 +48,7 @@ create or replace package ut_compound_data_helper authid definer is type tt_row_diffs is table of t_row_diffs; - function get_columns_info(a_cursor in out nocopy sys_refcursor) return xmltype; + function get_column_info_xml(a_column_details ut_key_anyval_pair) return xmltype; function get_columns_filter( a_exclude_xpath varchar2, a_include_xpath varchar2, diff --git a/source/expectations/data_values/ut_curr_usr_compound_helper.pkb b/source/expectations/data_values/ut_curr_usr_compound_helper.pkb new file mode 100644 index 000000000..357a1122b --- /dev/null +++ b/source/expectations/data_values/ut_curr_usr_compound_helper.pkb @@ -0,0 +1,186 @@ +create or replace package body ut_curr_usr_compound_helper is + + type t_type_name_map is table of varchar2(100) index by binary_integer; + g_type_name_map t_type_name_map; + g_anytype_name_map t_type_name_map; + g_user_defined_type pls_integer := dbms_sql.user_defined_type; + + + function get_column_type(a_desc_rec dbms_sql.desc_rec3,a_desc_user_types boolean := false) return ut_key_anyval_pair is + l_result ut_key_anyval_pair; + l_data_type varchar2(500) := 'unknown datatype'; + begin + if g_type_name_map.exists(a_desc_rec.col_type) then + l_result := ut_key_varcharvalue_pair(a_desc_rec.col_name,g_type_name_map(a_desc_rec.col_type)); + elsif a_desc_rec.col_type = g_user_defined_type and a_desc_user_types then + l_result := ut_key_xmlvalue_pair(a_desc_rec.col_name, + get_user_defined_type(a_desc_rec.col_schema_name,a_desc_rec.col_type_name)); + elsif a_desc_rec.col_schema_name is not null and a_desc_rec.col_type_name is not null then + l_result := ut_key_varcharvalue_pair(a_desc_rec.col_name,a_desc_rec.col_schema_name||'.'||a_desc_rec.col_type_name); + end if; + return l_result; + end; + + function get_columns_info(a_columns_tab dbms_sql.desc_tab3, a_columns_count integer,a_desc_user_types boolean := false) return ut_key_anyval_pairs is + l_result ut_key_anyval_pairs := ut_key_anyval_pairs(); + begin + for i in 1 .. a_columns_count loop + l_result.extend; + l_result(l_result.last) := get_column_type(a_columns_tab(i),a_desc_user_types); + end loop; + return l_result; + end; + + function get_descr_cursor(a_cursor in out nocopy sys_refcursor,a_desc_user_types boolean := false) return ut_key_anyval_pairs is + l_cursor_number integer; + l_columns_count pls_integer; + l_columns_desc dbms_sql.desc_tab3; + l_columns_tab ut_key_anyval_pairs; + begin + if a_cursor is null or not a_cursor%isopen then + return null; + end if; + l_cursor_number := dbms_sql.to_cursor_number( a_cursor ); + dbms_sql.describe_columns3( l_cursor_number, l_columns_count, l_columns_desc ); + a_cursor := dbms_sql.to_refcursor( l_cursor_number ); + l_columns_tab := get_columns_info( l_columns_desc, l_columns_count,a_desc_user_types); + return l_columns_tab; + end; + + function get_columns_info(a_cursor in out nocopy sys_refcursor,a_desc_user_types boolean := false) return xmltype is + l_result xmltype; + l_result_tmp xmltype; + l_columns_tab ut_key_anyval_pairs; + begin + l_columns_tab := get_descr_cursor(a_cursor,a_desc_user_types); + + for i in 1..l_columns_tab.COUNT + loop + l_result_tmp := ut_compound_data_helper.get_column_info_xml(l_columns_tab(i)); + select xmlconcat(l_result,l_result_tmp) into l_result from dual; + end loop; + + select XMLELEMENT("ROW",l_result ) + into l_result from dual; + + return l_result; + end; + + function get_anytype_attribute_count (a_anytype anytype) return pls_integer is + l_attribute_typecode pls_integer; + l_schema_name varchar2(32767); + l_version varchar2(32767); + l_type_name varchar2(32767); + l_attributes pls_integer; + l_prec pls_integer; + l_scale pls_integer; + l_len pls_integer; + l_csid pls_integer; + l_csfrm pls_integer; + begin + l_attribute_typecode := a_anytype.getinfo( + prec => l_prec, + scale => l_scale, + len => l_len, + csid => l_csid, + csfrm => l_csfrm, + schema_name => l_schema_name, + type_name => l_type_name, + version => l_version, + numelems => l_attributes); + return l_attributes; + end; + + function get_anytype_attributes_info (a_anytype anytype) return ut_key_value_pairs is + l_result ut_key_value_pairs := ut_key_value_pairs(); + l_attribute_typecode pls_integer; + l_aname varchar2(32767); + l_prec pls_integer; + l_scale pls_integer; + l_len pls_integer; + l_csid pls_integer; + l_csfrm pls_integer; + l_attr_elt_type anytype; + begin + for i in 1..get_anytype_attribute_count(a_anytype) loop + l_attribute_typecode := a_anytype.getAttrElemInfo( + pos => i, --First attribute + prec => l_prec, + scale => l_scale, + len => l_len, + csid => l_csid, + csfrm => l_csfrm, + attr_elt_type => l_attr_elt_type, + aname => l_aname); + + l_result.extend; + l_result(l_result.last) := ut_key_value_pair(l_aname, g_anytype_name_map(l_attribute_typecode)); + + end loop; + return l_result; + end; + + function get_user_defined_type(a_owner varchar2,a_type_name varchar2) return xmltype is + l_anydata anydata; + l_anytype anytype; + l_typecode pls_integer; + l_result xmltype; + l_columns_tab ut_key_value_pairs := ut_key_value_pairs(); + begin + execute immediate 'declare + l_v '||a_owner||'.'||a_type_name||'; + begin + :anydata := anydata.convertobject(l_v); + end;' USING IN OUT l_anydata; + + l_typecode := l_anydata.gettype(l_anytype); + l_columns_tab := get_anytype_attributes_info(l_anytype); + + select xmlagg(xmlelement(evalname key,value)) + into l_result from table(l_columns_tab); + + return l_result; + + end; + + begin + g_anytype_name_map(dbms_types.typecode_date) :=' DATE'; + g_anytype_name_map(dbms_types.typecode_number) := 'NUMBER'; + g_anytype_name_map(dbms_types.typecode_raw) := 'RAW'; + g_anytype_name_map(dbms_types.typecode_char) := 'CHAR'; + g_anytype_name_map(dbms_types.typecode_varchar2) := 'VARCHAR2'; + g_anytype_name_map(dbms_types.typecode_varchar) := 'VARCHAR'; + g_anytype_name_map(dbms_types.typecode_blob) := 'BLOB'; + g_anytype_name_map(dbms_types.typecode_bfile) := 'BFILE'; + g_anytype_name_map(dbms_types.typecode_clob) := 'CLOB'; + g_anytype_name_map(dbms_types.typecode_timestamp) := 'TIMESTAMP'; + g_anytype_name_map(dbms_types.typecode_timestamp_tz) := 'TIMESTAMP WITH TIME ZONE'; + g_anytype_name_map(dbms_types.typecode_timestamp_ltz) := 'TIMESTAMP WITH LOCAL TIME ZONE'; + g_anytype_name_map(dbms_types.typecode_interval_ym) := 'INTERVAL YEAR TO MONTH'; + g_anytype_name_map(dbms_types.typecode_interval_ds) := 'INTERVAL DAY TO SECOND'; + g_anytype_name_map(dbms_types.typecode_bfloat) := 'BINARY_FLOAT'; + g_anytype_name_map(dbms_types.typecode_bdouble) := 'BINARY_DOUBLE'; + g_anytype_name_map(dbms_types.typecode_urowid) := 'UROWID'; + + g_type_name_map( dbms_sql.binary_bouble_type ) := 'BINARY_DOUBLE'; + g_type_name_map( dbms_sql.bfile_type ) := 'BFILE'; + g_type_name_map( dbms_sql.binary_float_type ) := 'BINARY_FLOAT'; + g_type_name_map( dbms_sql.blob_type ) := 'BLOB'; + g_type_name_map( dbms_sql.long_raw_type ) := 'LONG RAW'; + g_type_name_map( dbms_sql.char_type ) := 'CHAR'; + g_type_name_map( dbms_sql.clob_type ) := 'CLOB'; + g_type_name_map( dbms_sql.long_type ) := 'LONG'; + g_type_name_map( dbms_sql.date_type ) := 'DATE'; + g_type_name_map( dbms_sql.interval_day_to_second_type ) := 'INTERVAL DAY TO SECOND'; + g_type_name_map( dbms_sql.interval_year_to_month_type ) := 'INTERVAL YEAR TO MONTH'; + g_type_name_map( dbms_sql.raw_type ) := 'RAW'; + g_type_name_map( dbms_sql.timestamp_type ) := 'TIMESTAMP'; + g_type_name_map( dbms_sql.timestamp_with_tz_type ) := 'TIMESTAMP WITH TIME ZONE'; + g_type_name_map( dbms_sql.timestamp_with_local_tz_type ) := 'TIMESTAMP WITH LOCAL TIME ZONE'; + g_type_name_map( dbms_sql.varchar2_type ) := 'VARCHAR2'; + g_type_name_map( dbms_sql.number_type ) := 'NUMBER'; + g_type_name_map( dbms_sql.rowid_type ) := 'ROWID'; + g_type_name_map( dbms_sql.urowid_type ) := 'UROWID'; + +end; +/ diff --git a/source/expectations/data_values/ut_curr_usr_compound_helper.pks b/source/expectations/data_values/ut_curr_usr_compound_helper.pks new file mode 100644 index 000000000..3955f49c8 --- /dev/null +++ b/source/expectations/data_values/ut_curr_usr_compound_helper.pks @@ -0,0 +1,8 @@ +create or replace package ut_curr_usr_compound_helper authid current_user is + + function get_columns_info(a_cursor in out nocopy sys_refcursor,a_desc_user_types boolean := false) return xmltype; + + function get_user_defined_type(a_owner varchar2,a_type_name varchar2) return xmltype; + +end; +/ diff --git a/source/expectations/data_values/ut_data_value_refcursor.tpb b/source/expectations/data_values/ut_data_value_refcursor.tpb index c36b37f69..fe8c1f443 100644 --- a/source/expectations/data_values/ut_data_value_refcursor.tpb +++ b/source/expectations/data_values/ut_data_value_refcursor.tpb @@ -37,7 +37,8 @@ create or replace type body ut_data_value_refcursor as self.data_type := 'refcursor'; if l_cursor is not null then if l_cursor%isopen then - self.columns_info := ut_compound_data_helper.get_columns_info(l_cursor); + self.columns_info := ut_curr_usr_compound_helper.get_columns_info(l_cursor); + self.key_info := ut_curr_usr_compound_helper.get_columns_info(l_cursor,true); self.elements_count := 0; -- We use DBMS_XMLGEN in order to: -- 1) be able to process data in bulks (set of rows) @@ -137,9 +138,9 @@ create or replace type body ut_data_value_refcursor as return case a_missing_keys.diff_type when 'a' then - ' Unknown key to join by in actual:'||a_missing_keys.missingxpath + ' Join key '||a_missing_keys.missingxpath||' does not exists in actual' when 'e' then - ' Unknown key to join by in expected:'||a_missing_keys.missingxpath + ' Join key '||a_missing_keys.missingxpath||' does not exists in expected' end; end; @@ -190,7 +191,7 @@ create or replace type body ut_data_value_refcursor as --check for missing pk if (a_join_by_xpath is not null) then - l_missing_pk := ut_compound_data_helper.is_pk_exists(self.columns_info, l_actual.columns_info, a_exclude_xpath, a_include_xpath,a_join_by_xpath); + l_missing_pk := ut_compound_data_helper.is_pk_exists(self.key_info, l_actual.key_info, a_exclude_xpath, a_include_xpath,a_join_by_xpath); end if; --diff rows and row elements if the pk is not missing @@ -239,7 +240,7 @@ create or replace type body ut_data_value_refcursor as l_other := treat(a_other as ut_data_value_refcursor); - l_missing_pk := ut_compound_data_helper.is_pk_exists(self.columns_info, l_other.columns_info, a_exclude_xpath, a_include_xpath,a_join_by_xpath); + l_missing_pk := ut_compound_data_helper.is_pk_exists(self.key_info, l_other.key_info, a_exclude_xpath, a_include_xpath,a_join_by_xpath); --if we join by key and key is missing fail and report error if a_join_by_xpath is not null and l_missing_pk.count > 0 then l_result := 1; diff --git a/source/expectations/data_values/ut_data_value_refcursor.tps b/source/expectations/data_values/ut_data_value_refcursor.tps index af0609bb6..c73aaa538 100644 --- a/source/expectations/data_values/ut_data_value_refcursor.tps +++ b/source/expectations/data_values/ut_data_value_refcursor.tps @@ -29,7 +29,12 @@ create or replace type ut_data_value_refcursor under ut_compound_data_value( * Holds information about column names and column data-types */ columns_info xmltype, - + + /** + * Holds more detailed information regarding the pk joins + */ + key_info xmltype, + constructor function ut_data_value_refcursor(self in out nocopy ut_data_value_refcursor, a_value sys_refcursor) return self as result, member procedure init(self in out nocopy ut_data_value_refcursor, a_value sys_refcursor), overriding member function to_string return varchar2, @@ -38,4 +43,4 @@ create or replace type ut_data_value_refcursor under ut_compound_data_value( overriding member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean) return integer ) -/ \ No newline at end of file +/ diff --git a/source/install.sql b/source/install.sql index 3050349e6..6d7bba6d9 100644 --- a/source/install.sql +++ b/source/install.sql @@ -43,6 +43,10 @@ alter session set current_schema = &&ut3_owner; @@install_component.sql 'core/types/ut_object_names.tps' @@install_component.sql 'core/types/ut_key_value_pair.tps' @@install_component.sql 'core/types/ut_key_value_pairs.tps' +@@install_component.sql 'core/types/ut_key_anyval_pair.tps' +@@install_component.sql 'core/types/ut_key_anyval_pairs.tps' +@@install_component.sql 'core/types/ut_key_varcharvalue_pair.tps' +@@install_component.sql 'core/types/ut_key_xmlvalue_pair.tps' @@install_component.sql 'core/ut_utils.pks' @@install_component.sql 'core/ut_metadata.pks' @@install_component.sql 'core/ut_utils.pkb' @@ -182,6 +186,7 @@ prompt Installing DBMSPLSQL Tables objects into &&ut3_owner schema @@install_component.sql 'expectations/data_values/ut_data_value_varchar2.tps' @@install_component.sql 'expectations/data_values/ut_data_value_yminterval.tps' @@install_component.sql 'expectations/data_values/ut_compound_data_helper.pks' +@@install_component.sql 'expectations/data_values/ut_curr_usr_compound_helper.pks' @@install_component.sql 'expectations/matchers/ut_matcher.tps' @@install_component.sql 'expectations/matchers/ut_comparison_matcher.tps' @@install_component.sql 'expectations/matchers/ut_be_false.tps' @@ -203,6 +208,7 @@ prompt Installing DBMSPLSQL Tables objects into &&ut3_owner schema @@install_component.sql 'expectations/data_values/ut_data_value.tpb' @@install_component.sql 'expectations/data_values/ut_compound_data_value.tpb' @@install_component.sql 'expectations/data_values/ut_compound_data_helper.pkb' +@@install_component.sql 'expectations/data_values/ut_curr_usr_compound_helper.pkb' @@install_component.sql 'expectations/data_values/ut_data_value_anydata.tpb' @@install_component.sql 'expectations/data_values/ut_data_value_object.tpb' @@install_component.sql 'expectations/data_values/ut_data_value_collection.tpb' diff --git a/source/uninstall.sql b/source/uninstall.sql index 7b2ad4f66..d8796464c 100644 --- a/source/uninstall.sql +++ b/source/uninstall.sql @@ -260,6 +260,14 @@ drop package ut_event_manager; drop type ut_event_item force; +drop type ut_key_anyval_pair force; + +drop type ut_key_anyval_pairs force; + +drop type ut_key_varcharvalue_pair force; + +drop type ut_key_xmlvalue_pair force; + drop type ut_key_value_pairs force; drop type ut_key_value_pair force; diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pkb b/test/core/expectations/compound_data/test_expectations_cursor.pkb index 12602c61f..489732947 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pkb +++ b/test/core/expectations/compound_data/test_expectations_cursor.pkb @@ -1130,8 +1130,8 @@ Extra: test-666%]'; l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% Diff:% %Unable to join sets:% -%Unknown key to join by in expected:/*/OWNER% -%Unknown key to join by in actual:/*/OWNER%]'; +%Join key /*/OWNER does not exists in expected% +%Join key /*/OWNER does not exists in actual%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1153,10 +1153,10 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% Diff:% %Unable to join sets:% -%Unknown key to join by in expected:/*/OWNER% -%Unknown key to join by in expected:/*/USER_ID% -%Unknown key to join by in actual:/*/OWNER% -%Unknown key to join by in actual:/*/USER_ID%]'; +%Join key /*/OWNER does not exists in expected% +%Join key /*/USER_ID does not exists in expected% +%Join key /*/OWNER does not exists in actual% +%Join key /*/USER_ID does not exists in actual%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1178,8 +1178,8 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% Diff:% %Unable to join sets:% -%Unknown key to join by in expected:/*/SOME_COL% -%Unknown key to join by in actual:/*/SOME_COL%]'; +%Join key /*/SOME_COL does not exists in expected% +%Join key /*/SOME_COL does not exists in actual%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1201,8 +1201,8 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% Diff:% %Unable to join sets:% -%Unknown key to join by in expected:/*/RN% -%Unknown key to join by in actual:/*/RN%]'; +%Join key /*/RN does not exists in expected% +%Join key /*/RN does not exists in actual%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1224,7 +1224,7 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% Diff:% %Unable to join sets:% -%Unknown key to join by in expected:/*/RNI%]'; +%Join key /*/RNI does not exists in expected%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1246,7 +1246,7 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% Diff:% %Unable to join sets:% -%Unknown key to join by in actual:/*/RNI%]'; +%Join key /*/RNI does not exists in actual%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1440,6 +1440,117 @@ Diff:% --Assert ut.expect(expectations.failed_expectations_data()).to_be_empty(); end; + + procedure compare_nested_tab_col_un is + l_actual sys_refcursor; + l_expected sys_refcursor; + begin + --Arrange + open l_actual for select test_dummy_object( rownum, 'Something '||rownum, rownum) as colval + from dual connect by level <=2 order by rownum asc; + + open l_expected for select test_dummy_object( rownum, 'Something '||rownum, rownum) as colval + from dual connect by level <=2 order by rownum desc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).unordered; + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + end; + + procedure compare_nested_tab_col_jb is + l_actual sys_refcursor; + l_expected sys_refcursor; + begin + --Arrange + open l_actual for select test_dummy_object( rownum, 'Something '||rownum, rownum) as colval + from dual connect by level <=2 order by rownum asc; + + open l_expected for select test_dummy_object( rownum, 'Something '||rownum, rownum) as colval + from dual connect by level <=2 order by rownum desc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by('COLVAL/ID'); + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + end; + + procedure comp_nest_tab_col_un_fail is + l_actual sys_refcursor; + l_expected sys_refcursor; + begin + --Arrange + open l_actual for select test_dummy_object( rownum, 'Something '||rownum, rownum) as colval + from dual connect by level <=2 order by rownum asc; + + open l_expected for select test_dummy_object( rownum, 'Somethings '||rownum, rownum) as colval + from dual connect by level <=3 order by rownum desc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).unordered; + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_not_null(); + end; + + procedure comp_nest_tab_col_jb_fail is + l_actual sys_refcursor; + l_expected sys_refcursor; + begin + --Arrange + open l_actual for select test_dummy_object( rownum, 'Something '||rownum, rownum) as colval + from dual connect by level <=2 order by rownum asc; + + open l_expected for select test_dummy_object( rownum, 'Somethings '||rownum, rownum) as colval + from dual connect by level <=2 order by rownum desc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by('COLVAL/ID'); + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_not_null(); + end; + + procedure comp_nest_tab_col_jb_multi is + l_actual sys_refcursor; + l_expected sys_refcursor; + begin + --Arrange + open l_actual for select rownum as rn,test_dummy_object( rownum, 'Something '||rownum, rownum) as colval + from dual connect by level <=2 order by rownum asc; + + open l_expected for select rownum as rn,test_dummy_object( rownum, 'Something '||rownum, rownum) as colval + from dual connect by level <=2 order by rownum desc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by(ut3.ut_varchar2_list('RN,COLVAL/ID')); + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + end; + procedure comp_nest_tab_col_jb_nokey is + l_actual sys_refcursor; + l_expected sys_refcursor; + l_expected_message varchar2(32767); + l_actual_message varchar2(32767); + begin + --Arrange + open l_actual for select test_dummy_object( rownum, 'Something '||rownum, rownum) as colval + from dual connect by level <=2 order by rownum asc; + + open l_expected for select test_dummy_object( rownum, 'Something '||rownum, rownum) as colval + from dual connect by level <=2 order by rownum desc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by('COLVAL/IDS'); + --Assert + --Assert + l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 2 ]% +Diff:% +%Unable to join sets:% +%Join key /*/COLVAL/IDS does not exists in expected% +%Join key /*/COLVAL/IDS does not exists in actual%]'; + l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; + --Assert + ut.expect(l_actual_message).to_be_like(l_expected_message); + end; end; / diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pks b/test/core/expectations/compound_data/test_expectations_cursor.pks index eb5144c29..340047d3a 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pks +++ b/test/core/expectations/compound_data/test_expectations_cursor.pks @@ -26,7 +26,7 @@ create or replace package test_expectations_cursor is --%test(Gives success on to_be_null if cursor is null) procedure success_to_be_null; - --%test(Gives succes on not_to_be_not_null if cursor is null) + --%test(Gives success on not_to_be_not_null if cursor is null) procedure success_not_to_be_not_null; --%test(Gives success on not_to_be_null if cursor is not null) @@ -262,5 +262,23 @@ create or replace package test_expectations_cursor is --%test(Include column of same type leaving different type out and exclude different type) procedure inlc_exc_dif_cols_as_list; + --%test(Compare nested table type unordered) + procedure compare_nested_tab_col_un; + + --%test(Compare nested table type join by) + procedure compare_nested_tab_col_jb; + + --%test(Compare nested table type unordered fail) + procedure comp_nest_tab_col_un_fail; + + --%test(Compare nested table type join by fail) + procedure comp_nest_tab_col_jb_fail; + + --%test(Compare nested table type join by multi key) + procedure comp_nest_tab_col_jb_multi; + + --%test(Compare nested table type join by missing nested key) + procedure comp_nest_tab_col_jb_nokey; + end; / From 77e446830a12353b91877817670942ed56a7fcad Mon Sep 17 00:00:00 2001 From: lwasylow Date: Tue, 22 May 2018 07:00:21 +0100 Subject: [PATCH 18/24] Updated code with a new display update tests to include table type Update key value pairs type --- docs/userguide/advanced_data_comparison.md | 8 +- source/core/types/ut_key_xmlvalue_pair.tps | 20 -- .../data_values/ut_compound_data_helper.pkb | 35 ++- .../data_values/ut_compound_data_helper.pks | 2 +- .../data_values/ut_compound_data_value.tpb | 2 +- .../ut_curr_usr_compound_helper.pkb | 12 +- .../data_values/ut_data_value_xmltype.tpb | 50 +++++ .../data_values/ut_data_value_xmltype.tps} | 8 +- .../data_values}/ut_key_anyval_pair.tps | 3 +- .../data_values}/ut_key_anyval_pairs.tps | 0 source/install.sql | 8 +- source/uninstall.sql | 6 +- .../test_expectations_cursor.pkb | 204 +++++++++++++++--- .../test_expectations_cursor.pks | 34 ++- 14 files changed, 292 insertions(+), 100 deletions(-) delete mode 100644 source/core/types/ut_key_xmlvalue_pair.tps create mode 100644 source/expectations/data_values/ut_data_value_xmltype.tpb rename source/{core/types/ut_key_varcharvalue_pair.tps => expectations/data_values/ut_data_value_xmltype.tps} (59%) rename source/{core/types => expectations/data_values}/ut_key_anyval_pair.tps (93%) rename source/{core/types => expectations/data_values}/ut_key_anyval_pairs.tps (100%) diff --git a/docs/userguide/advanced_data_comparison.md b/docs/userguide/advanced_data_comparison.md index 1edc39b7c..cc488da11 100644 --- a/docs/userguide/advanced_data_comparison.md +++ b/docs/userguide/advanced_data_comparison.md @@ -163,10 +163,14 @@ This will show you difference in row 'TEST' regardless of order. ```sql Rows: [ 1 differences ] - Expected: -600 for key: TEST - Actual: -610 for key: TEST + PK TEST - Expected: -600 + PK TEST - Actual: -610 ``` +Assumption is that join by is made by column name so that what will be displayed as part of results. + + + **Please note that .join_by option will take longer to process due to need of parsing via primary keys.** ## Defining item as XPath diff --git a/source/core/types/ut_key_xmlvalue_pair.tps b/source/core/types/ut_key_xmlvalue_pair.tps deleted file mode 100644 index 1f5462861..000000000 --- a/source/core/types/ut_key_xmlvalue_pair.tps +++ /dev/null @@ -1,20 +0,0 @@ -create or replace type ut_key_xmlvalue_pair under ut_key_anyval_pair ( - /* - utPLSQL - Version 3 - Copyright 2016 - 2017 utPLSQL Project - - Licensed under the Apache License, Version 2.0 (the "License"): - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - value xmltype -) -/ diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index 71963263f..a89258c19 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -21,15 +21,17 @@ create or replace package body ut_compound_data_helper is function get_column_info_xml(a_column_details ut_key_anyval_pair) return xmltype is l_result varchar2(4000); l_res xmltype; + l_data ut_data_value := a_column_details.value; + l_key varchar2(4000) := ut_utils.xmlgen_escaped_string(a_column_details.KEY); begin - l_result := '<'||ut_utils.xmlgen_escaped_string(a_column_details.KEY)||' xml_valid_name="'||ut_utils.xmlgen_escaped_string(a_column_details.KEY)||'">'; - if a_column_details is of(ut_key_xmlvalue_pair) then - l_result := l_result || (treat(a_column_details as ut_key_xmlvalue_pair).value.getstringval()); + l_result := '<'||l_key||' xml_valid_name="'||l_key||'">'; + if l_data is of(ut_data_value_xmltype) then + l_result := l_result || (treat(l_data as ut_data_value_xmltype).to_string); else - l_result := l_result || ut_utils.xmlgen_escaped_string((treat(a_column_details as ut_key_varcharvalue_pair).value)); + l_result := l_result || ut_utils.xmlgen_escaped_string((treat(l_data as ut_data_value_varchar2).data_value)); end if; - l_result := l_result ||''; + l_result := l_result ||''; return xmltype(l_result); end; @@ -155,16 +157,13 @@ create or replace package body ut_compound_data_helper is return l_results; end; - function get_pk_value (a_join_by_xpath varchar2,a_item_data xmltype) return varchar2 is - l_pk_value varchar2(4000); + function get_pk_value (a_join_by_xpath varchar2,a_item_data xmltype) return clob is + l_pk_value clob; begin - select listagg(extractvalue(xmlelement("ROW",column_value),a_join_by_xpath),':') within group ( order by 1) - into l_pk_value - from table(xmlsequence(extract(a_item_data,'/*/*'))); - - return l_pk_value; + select extract(a_item_data,a_join_by_xpath).getclobval() into l_pk_value from dual; + return l_pk_value; exception when no_data_found then - return 'null '; + return 'null'; end; function get_rows_diff( @@ -242,11 +241,11 @@ create or replace package body ut_compound_data_helper is unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:') ) ) union all - select case when exp.pk_hash is null then 'Extra:' else 'Missing:' end as diff_type, + select case when exp.pk_hash is null then 'Extra' else 'Missing' end as diff_type, xmlserialize(content nvl(exp.item_data, act.item_data) no indent) diffed_row, coalesce(exp.pk_hash,act.pk_hash) pk_hash, coalesce(exp.pk_value,act.pk_value) pk_value - from (select extract(ucd.item_data,'/*/*') item_data,i.pk_hash, + from (select extract(deletexml(ucd.item_data, :join_by),'/*/*') item_data,i.pk_hash, ut_compound_data_helper.get_pk_value(:join_by,item_data) pk_value from ut_compound_data_tmp ucd, diff_info i @@ -254,7 +253,7 @@ create or replace package body ut_compound_data_helper is and ucd.item_hash = i.item_hash ) exp full outer join ( - select extract(ucd.item_data,'/*/*') item_data,i.pk_hash, + select extract(deletexml(ucd.item_data, :join_by),'/*/*') item_data,i.pk_hash, ut_compound_data_helper.get_pk_value(:join_by,item_data) pk_value from ut_compound_data_tmp ucd, diff_info i @@ -272,7 +271,7 @@ create or replace package body ut_compound_data_helper is a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, a_join_by_xpath, a_exclude_xpath, a_include_xpath, a_actual_dataset_guid, - a_join_by_xpath,a_expected_dataset_guid,a_join_by_xpath, a_actual_dataset_guid, + a_join_by_xpath,a_join_by_xpath,a_expected_dataset_guid,a_join_by_xpath,a_join_by_xpath, a_actual_dataset_guid, a_max_rows; return l_results; @@ -502,7 +501,7 @@ create or replace package body ut_compound_data_helper is with xpaths_tab as (select column_value xpath from table(:xpath_tabs)), expected_column_info as ( select :expected as item_data from dual ), actual_column_info as ( select :actual as item_data from dual ) - select xpath,diif_type from + select REGEXP_SUBSTR (xpath,'[^(/\*/)](.+)$'),diif_type from ( (select xpath,'e' diif_type from xpaths_tab minus diff --git a/source/expectations/data_values/ut_compound_data_helper.pks b/source/expectations/data_values/ut_compound_data_helper.pks index 6558182ae..da2863667 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pks +++ b/source/expectations/data_values/ut_compound_data_helper.pks @@ -59,7 +59,7 @@ create or replace package ut_compound_data_helper authid definer is a_expected xmltype, a_actual xmltype, a_exclude_xpath varchar2, a_include_xpath varchar2 ) return tt_column_diffs; - function get_pk_value (a_join_by_xpath varchar2,a_item_data xmltype) return varchar2; + function get_pk_value (a_join_by_xpath varchar2,a_item_data xmltype) return clob; function compare_type(a_join_by_xpath in varchar2,a_unordered boolean) return varchar2; diff --git a/source/expectations/data_values/ut_compound_data_value.tpb b/source/expectations/data_values/ut_compound_data_value.tpb index 76f22e60f..45841e5a9 100644 --- a/source/expectations/data_values/ut_compound_data_value.tpb +++ b/source/expectations/data_values/ut_compound_data_value.tpb @@ -97,7 +97,7 @@ create or replace type body ut_compound_data_value as function get_diff_message (a_row_diff ut_compound_data_helper.t_row_diffs,a_compare_type varchar2) return varchar2 is begin if a_compare_type = ut_compound_data_helper.gc_compare_join_by and a_row_diff.pk_value is not null then - return ' '||rpad(a_row_diff.diff_type,10)||a_row_diff.diffed_row||' for key: '||a_row_diff.pk_value; + return ' PK '||a_row_diff.pk_value||' - '||rpad(a_row_diff.diff_type,10)||a_row_diff.diffed_row; elsif a_compare_type = ut_compound_data_helper.gc_compare_join_by or a_compare_type = ut_compound_data_helper.gc_compare_normal then return ' Row No. '||a_row_diff.rn||' - '||rpad(a_row_diff.diff_type,10)||a_row_diff.diffed_row; elsif a_compare_type = ut_compound_data_helper.gc_compare_unordered then diff --git a/source/expectations/data_values/ut_curr_usr_compound_helper.pkb b/source/expectations/data_values/ut_curr_usr_compound_helper.pkb index 357a1122b..7885b15fe 100644 --- a/source/expectations/data_values/ut_curr_usr_compound_helper.pkb +++ b/source/expectations/data_values/ut_curr_usr_compound_helper.pkb @@ -7,18 +7,18 @@ create or replace package body ut_curr_usr_compound_helper is function get_column_type(a_desc_rec dbms_sql.desc_rec3,a_desc_user_types boolean := false) return ut_key_anyval_pair is + l_data ut_data_value; l_result ut_key_anyval_pair; l_data_type varchar2(500) := 'unknown datatype'; - begin + begin if g_type_name_map.exists(a_desc_rec.col_type) then - l_result := ut_key_varcharvalue_pair(a_desc_rec.col_name,g_type_name_map(a_desc_rec.col_type)); + l_data := ut_data_value_varchar2(g_type_name_map(a_desc_rec.col_type)); elsif a_desc_rec.col_type = g_user_defined_type and a_desc_user_types then - l_result := ut_key_xmlvalue_pair(a_desc_rec.col_name, - get_user_defined_type(a_desc_rec.col_schema_name,a_desc_rec.col_type_name)); + l_data :=ut_data_value_xmltype(get_user_defined_type(a_desc_rec.col_schema_name,a_desc_rec.col_type_name)); elsif a_desc_rec.col_schema_name is not null and a_desc_rec.col_type_name is not null then - l_result := ut_key_varcharvalue_pair(a_desc_rec.col_name,a_desc_rec.col_schema_name||'.'||a_desc_rec.col_type_name); + l_data := ut_data_value_varchar2(a_desc_rec.col_schema_name||'.'||a_desc_rec.col_type_name); end if; - return l_result; + return ut_key_anyval_pair(a_desc_rec.col_name,l_data); end; function get_columns_info(a_columns_tab dbms_sql.desc_tab3, a_columns_count integer,a_desc_user_types boolean := false) return ut_key_anyval_pairs is diff --git a/source/expectations/data_values/ut_data_value_xmltype.tpb b/source/expectations/data_values/ut_data_value_xmltype.tpb new file mode 100644 index 000000000..237a7e592 --- /dev/null +++ b/source/expectations/data_values/ut_data_value_xmltype.tpb @@ -0,0 +1,50 @@ +create or replace type body ut_data_value_xmltype as + /* + utPLSQL - Version 3 + Copyright 2016 - 2017 utPLSQL Project + + Licensed under the Apache License, Version 2.0 (the "License"): + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + + constructor function ut_data_value_xmltype(self in out nocopy ut_data_value_xmltype, a_value xmltype) return self as result is + begin + self.data_value := a_value; + self.self_type := $$plsql_unit; + self.data_type := 'xmltype'; + return; + end; + + overriding member function is_null return boolean is + begin + return (self.data_value is null); + end; + + overriding member function to_string return varchar2 is + begin + return ut_utils.to_string(self.data_value.getClobVal()); + end; + + overriding member function compare_implementation(a_other ut_data_value) return integer is + l_result integer; + l_other ut_data_value_xmltype; + begin + if a_other is of (ut_data_value_xmltype) then + l_other := treat(a_other as ut_data_value_xmltype); + l_result := dbms_lob.compare(self.data_value.getClobVal(),l_other.data_value.getClobVal()); + end if; + + return l_result; + end; + +end; +/ diff --git a/source/core/types/ut_key_varcharvalue_pair.tps b/source/expectations/data_values/ut_data_value_xmltype.tps similarity index 59% rename from source/core/types/ut_key_varcharvalue_pair.tps rename to source/expectations/data_values/ut_data_value_xmltype.tps index fdce93cf0..2bc4ae985 100644 --- a/source/core/types/ut_key_varcharvalue_pair.tps +++ b/source/expectations/data_values/ut_data_value_xmltype.tps @@ -1,4 +1,4 @@ -create or replace type ut_key_varcharvalue_pair under ut_key_anyval_pair ( +create or replace type ut_data_value_xmltype under ut_data_value( /* utPLSQL - Version 3 Copyright 2016 - 2017 utPLSQL Project @@ -15,6 +15,10 @@ create or replace type ut_key_varcharvalue_pair under ut_key_anyval_pair ( See the License for the specific language governing permissions and limitations under the License. */ - value varchar2(4000) + data_value xmltype, + constructor function ut_data_value_xmltype(self in out nocopy ut_data_value_xmltype, a_value xmltype) return self as result, + overriding member function is_null return boolean, + overriding member function to_string return varchar2, + overriding member function compare_implementation(a_other ut_data_value) return integer ) / diff --git a/source/core/types/ut_key_anyval_pair.tps b/source/expectations/data_values/ut_key_anyval_pair.tps similarity index 93% rename from source/core/types/ut_key_anyval_pair.tps rename to source/expectations/data_values/ut_key_anyval_pair.tps index 5188bdda5..8fb4831c0 100644 --- a/source/core/types/ut_key_anyval_pair.tps +++ b/source/expectations/data_values/ut_key_anyval_pair.tps @@ -15,6 +15,7 @@ create or replace type ut_key_anyval_pair force as object( See the License for the specific language governing permissions and limitations under the License. */ - key varchar2(4000) + key varchar2(4000), + value ut_data_value ) not final / diff --git a/source/core/types/ut_key_anyval_pairs.tps b/source/expectations/data_values/ut_key_anyval_pairs.tps similarity index 100% rename from source/core/types/ut_key_anyval_pairs.tps rename to source/expectations/data_values/ut_key_anyval_pairs.tps diff --git a/source/install.sql b/source/install.sql index 6d7bba6d9..0bd30222c 100644 --- a/source/install.sql +++ b/source/install.sql @@ -43,10 +43,6 @@ alter session set current_schema = &&ut3_owner; @@install_component.sql 'core/types/ut_object_names.tps' @@install_component.sql 'core/types/ut_key_value_pair.tps' @@install_component.sql 'core/types/ut_key_value_pairs.tps' -@@install_component.sql 'core/types/ut_key_anyval_pair.tps' -@@install_component.sql 'core/types/ut_key_anyval_pairs.tps' -@@install_component.sql 'core/types/ut_key_varcharvalue_pair.tps' -@@install_component.sql 'core/types/ut_key_xmlvalue_pair.tps' @@install_component.sql 'core/ut_utils.pks' @@install_component.sql 'core/ut_metadata.pks' @@install_component.sql 'core/ut_utils.pkb' @@ -166,6 +162,8 @@ prompt Installing DBMSPLSQL Tables objects into &&ut3_owner schema @@install_component.sql 'core/types/ut_console_reporter_base.tpb' --expectations and matchers +@@install_component.sql 'expectations/data_values/ut_key_anyval_pair.tps' +@@install_component.sql 'expectations/data_values/ut_key_anyval_pairs.tps' @@install_component.sql 'expectations/data_values/ut_compound_data_tmp.sql' @@install_component.sql 'expectations/data_values/ut_compound_data_diff_tmp.sql' @@install_component.sql 'expectations/data_values/ut_data_value.tps' @@ -185,6 +183,7 @@ prompt Installing DBMSPLSQL Tables objects into &&ut3_owner schema @@install_component.sql 'expectations/data_values/ut_data_value_timestamp_ltz.tps' @@install_component.sql 'expectations/data_values/ut_data_value_varchar2.tps' @@install_component.sql 'expectations/data_values/ut_data_value_yminterval.tps' +@@install_component.sql 'expectations/data_values/ut_data_value_xmltype.tps' @@install_component.sql 'expectations/data_values/ut_compound_data_helper.pks' @@install_component.sql 'expectations/data_values/ut_curr_usr_compound_helper.pks' @@install_component.sql 'expectations/matchers/ut_matcher.tps' @@ -224,6 +223,7 @@ prompt Installing DBMSPLSQL Tables objects into &&ut3_owner schema @@install_component.sql 'expectations/data_values/ut_data_value_timestamp_ltz.tpb' @@install_component.sql 'expectations/data_values/ut_data_value_varchar2.tpb' @@install_component.sql 'expectations/data_values/ut_data_value_yminterval.tpb' +@@install_component.sql 'expectations/data_values/ut_data_value_xmltype.tpb' @@install_component.sql 'expectations/matchers/ut_matcher.tpb' @@install_component.sql 'expectations/matchers/ut_comparison_matcher.tpb' @@install_component.sql 'expectations/matchers/ut_be_false.tpb' diff --git a/source/uninstall.sql b/source/uninstall.sql index d8796464c..0a8540592 100644 --- a/source/uninstall.sql +++ b/source/uninstall.sql @@ -152,6 +152,8 @@ drop type ut_data_value_collection force; drop type ut_data_value_anydata force; +drop type ut_data_value_xmltype force; + drop type ut_data_value force; drop table ut_compound_data_tmp; @@ -264,10 +266,6 @@ drop type ut_key_anyval_pair force; drop type ut_key_anyval_pairs force; -drop type ut_key_varcharvalue_pair force; - -drop type ut_key_xmlvalue_pair force; - drop type ut_key_value_pairs force; drop type ut_key_value_pair force; diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pkb b/test/core/expectations/compound_data/test_expectations_cursor.pkb index 489732947..7a87aab87 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pkb +++ b/test/core/expectations/compound_data/test_expectations_cursor.pkb @@ -1130,8 +1130,8 @@ Extra: test-666%]'; l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% Diff:% %Unable to join sets:% -%Join key /*/OWNER does not exists in expected% -%Join key /*/OWNER does not exists in actual%]'; +%Join key OWNER does not exists in expected% +%Join key OWNER does not exists in actual%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1153,10 +1153,10 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% Diff:% %Unable to join sets:% -%Join key /*/OWNER does not exists in expected% -%Join key /*/USER_ID does not exists in expected% -%Join key /*/OWNER does not exists in actual% -%Join key /*/USER_ID does not exists in actual%]'; +%Join key OWNER does not exists in expected% +%Join key USER_ID does not exists in expected% +%Join key OWNER does not exists in actual% +%Join key USER_ID does not exists in actual%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1178,8 +1178,8 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% Diff:% %Unable to join sets:% -%Join key /*/SOME_COL does not exists in expected% -%Join key /*/SOME_COL does not exists in actual%]'; +%Join key SOME_COL does not exists in expected% +%Join key SOME_COL does not exists in actual%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1201,8 +1201,8 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% Diff:% %Unable to join sets:% -%Join key /*/RN does not exists in expected% -%Join key /*/RN does not exists in actual%]'; +%Join key RN does not exists in expected% +%Join key RN does not exists in actual%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1224,7 +1224,7 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% Diff:% %Unable to join sets:% -%Join key /*/RNI does not exists in expected%]'; +%Join key RNI does not exists in expected%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1246,7 +1246,7 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 3 ] was expected to equal: refcursor [ count = 3 ]% Diff:% %Unable to join sets:% -%Join key /*/RNI does not exists in actual%]'; +%Join key RNI does not exists in actual%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1284,8 +1284,8 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = % ] was expected to equal: refcursor [ count = % ] %Diff:% %Rows: [ 1 differences ]% -%Expected: -600 for key: TEST% -%Actual: -610 for key: TEST%]'; +%PK TEST - Expected:%-600% +%PK TEST - Actual:%-610%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1310,8 +1310,8 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = % ] was expected to equal: refcursor [ count = % ] %Diff:% %Rows: [ 2 differences ]% -%Missing: TEST-600 for key: -600:TEST% -%Extra: TEST-610 for key: -610:TEST%]'; +%PK TEST-600 - Missing % +%PK TEST-610 - Extra %]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1336,8 +1336,8 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = % ] was expected to equal: refcursor [ count = % ] %Diff:% %Rows: [ 1 differences ]% -%Expected: -600 for key: TEST:Y% -%Actual: -610 for key: TEST:Y%]'; +%PK TESTY - Expected:%-600% +%PK TESTY - Actual:%-610%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1441,7 +1441,7 @@ Diff:% ut.expect(expectations.failed_expectations_data()).to_be_empty(); end; - procedure compare_nested_tab_col_un is + procedure compare_obj_typ_col_un is l_actual sys_refcursor; l_expected sys_refcursor; begin @@ -1458,7 +1458,7 @@ Diff:% ut.expect(expectations.failed_expectations_data()).to_be_empty(); end; - procedure compare_nested_tab_col_jb is + procedure compare_obj_typ_col_jb is l_actual sys_refcursor; l_expected sys_refcursor; begin @@ -1475,9 +1475,11 @@ Diff:% ut.expect(expectations.failed_expectations_data()).to_be_empty(); end; - procedure comp_nest_tab_col_un_fail is + procedure comp_obj_typ_col_un_fail is l_actual sys_refcursor; l_expected sys_refcursor; + l_expected_message varchar2(32767); + l_actual_message varchar2(32767); begin --Arrange open l_actual for select test_dummy_object( rownum, 'Something '||rownum, rownum) as colval @@ -1488,11 +1490,20 @@ Diff:% --Act ut3.ut.expect(l_actual).to_equal(l_expected).unordered; + l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 3 ]% +Diff:% +Rows: [ 5 differences ] +Missing: 1Somethings 11% +Missing: 2Somethings 22% +Missing: 3Somethings 33% +Extra: 1Something 11% +Extra: 2Something 22%]'; + l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert - ut.expect(expectations.failed_expectations_data()).to_be_not_null(); + ut.expect(l_actual_message).to_be_like(l_expected_message); end; - procedure comp_nest_tab_col_jb_fail is + procedure comp_obj_typ_col_jb_fail is l_actual sys_refcursor; l_expected sys_refcursor; begin @@ -1509,7 +1520,7 @@ Diff:% ut.expect(expectations.failed_expectations_data()).to_be_not_null(); end; - procedure comp_nest_tab_col_jb_multi is + procedure comp_obj_typ_col_jb_multi is l_actual sys_refcursor; l_expected sys_refcursor; begin @@ -1526,7 +1537,7 @@ Diff:% ut.expect(expectations.failed_expectations_data()).to_be_empty(); end; - procedure comp_nest_tab_col_jb_nokey is + procedure comp_obj_typ_col_jb_nokey is l_actual sys_refcursor; l_expected sys_refcursor; l_expected_message varchar2(32767); @@ -1540,17 +1551,150 @@ Diff:% from dual connect by level <=2 order by rownum desc; --Act - ut3.ut.expect(l_actual).to_equal(l_expected).join_by('COLVAL/IDS'); - --Assert - --Assert + ut3.ut.expect(l_actual).to_equal(l_expected).join_by('COLVAL/IDS'); + l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 2 ]% Diff:% %Unable to join sets:% -%Join key /*/COLVAL/IDS does not exists in expected% -%Join key /*/COLVAL/IDS does not exists in actual%]'; +%Join key COLVAL/IDS does not exists in expected% +%Join key COLVAL/IDS does not exists in actual%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); end; + + procedure compare_nest_tab_col_jb is + l_actual sys_refcursor; + l_expected sys_refcursor; + l_actual_tab ut3.ut_key_value_pairs := ut3.ut_key_value_pairs(); + l_expected_tab ut3.ut_key_value_pairs := ut3.ut_key_value_pairs(); + l_expected_message varchar2(32767); + l_actual_message varchar2(32767); + begin + select ut3.ut_key_value_pair(rownum,'Something '||rownum) + bulk collect into l_actual_tab + from dual connect by level <=2; + + select ut3.ut_key_value_pair(rownum,'Something '||rownum) + bulk collect into l_expected_tab + from dual connect by level <=2; + + --Arrange + open l_actual for select key,value + from table(l_actual_tab) order by 1 asc; + + open l_expected for select key,value + from table(l_expected_tab) order by 1 desc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by('KEY'); + + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + end; + + procedure compare_nest_tab_col_jb_fail is + l_actual sys_refcursor; + l_expected sys_refcursor; + l_actual_tab ut3.ut_key_value_pairs := ut3.ut_key_value_pairs(); + l_expected_tab ut3.ut_key_value_pairs := ut3.ut_key_value_pairs(); + l_expected_message varchar2(32767); + l_actual_message varchar2(32767); + begin + select ut3.ut_key_value_pair(rownum,'Something '||rownum) + bulk collect into l_actual_tab + from dual connect by level <=2; + + select ut3.ut_key_value_pair(rownum,'Somethings '||rownum) + bulk collect into l_expected_tab + from dual connect by level <=2; + + --Arrange + open l_actual for select key,value + from table(l_actual_tab) order by 1 asc; + + open l_expected for select key,value + from table(l_expected_tab) order by 1 desc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by('KEY'); + l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 2 ]% +%Diff:% +%Rows: [ 2 differences ]% +%PK 1 - Expected: Somethings 1% +%PK 1 - Actual: Something 1% +%PK 2 - Expected: Somethings 2% +%PK 2 - Actual: Something 2%]'; + l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; + --Assert + ut.expect(l_actual_message).to_be_like(l_expected_message); + end; + + procedure compare_nest_tab_cols_jb is + l_actual sys_refcursor; + l_expected sys_refcursor; + l_actual_tab ut3.ut_key_value_pairs := ut3.ut_key_value_pairs(); + l_expected_tab ut3.ut_key_value_pairs := ut3.ut_key_value_pairs(); + l_expected_message varchar2(32767); + l_actual_message varchar2(32767); + begin + select ut3.ut_key_value_pair(rownum,'Something '||rownum) + bulk collect into l_actual_tab + from dual connect by level <=2; + + select ut3.ut_key_value_pair(rownum,'Something '||rownum) + bulk collect into l_expected_tab + from dual connect by level <=2; + + --Arrange + open l_actual for select key,value + from table(l_actual_tab) order by 1 asc; + + open l_expected for select key,value + from table(l_expected_tab) order by 1 desc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by(ut3.ut_varchar2_list('KEY,VALUE')); + + --Assert + ut.expect(expectations.failed_expectations_data()).to_be_empty(); + end; + + procedure compare_nest_tab_cols_jb_fail is + l_actual sys_refcursor; + l_expected sys_refcursor; + l_actual_tab ut3.ut_key_value_pairs := ut3.ut_key_value_pairs(); + l_expected_tab ut3.ut_key_value_pairs := ut3.ut_key_value_pairs(); + l_expected_message varchar2(32767); + l_actual_message varchar2(32767); + begin + select ut3.ut_key_value_pair(rownum,'Something '||rownum) + bulk collect into l_actual_tab + from dual connect by level <=2; + + select ut3.ut_key_value_pair(rownum,'Somethings '||rownum) + bulk collect into l_expected_tab + from dual connect by level <=2; + + --Arrange + open l_actual for select key,value + from table(l_actual_tab) order by 1 asc; + + open l_expected for select key,value + from table(l_expected_tab) order by 1 desc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by(ut3.ut_varchar2_list('KEY,VALUE')); + l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 2 ]% +%Diff:% +%Rows: [ 4 differences ]% +%PK 1Somethings 1 - Missing% +%PK 2Somethings 2 - Missing% +%PK 2Something 2 - Extra% +%PK 1Something 1 - Extra%]'; + l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; + --Assert + ut.expect(l_actual_message).to_be_like(l_expected_message); + end; end; / diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pks b/test/core/expectations/compound_data/test_expectations_cursor.pks index 340047d3a..185697cf8 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pks +++ b/test/core/expectations/compound_data/test_expectations_cursor.pks @@ -262,23 +262,35 @@ create or replace package test_expectations_cursor is --%test(Include column of same type leaving different type out and exclude different type) procedure inlc_exc_dif_cols_as_list; - --%test(Compare nested table type unordered) - procedure compare_nested_tab_col_un; + --%test(Compare object type unordered) + procedure compare_obj_typ_col_un; - --%test(Compare nested table type join by) - procedure compare_nested_tab_col_jb; + --%test(Compare object type join by) + procedure compare_obj_typ_col_jb; --%test(Compare nested table type unordered fail) - procedure comp_nest_tab_col_un_fail; + procedure comp_obj_typ_col_un_fail; - --%test(Compare nested table type join by fail) - procedure comp_nest_tab_col_jb_fail; + --%test(Compare object type join by fail) + procedure comp_obj_typ_col_jb_fail; - --%test(Compare nested table type join by multi key) - procedure comp_nest_tab_col_jb_multi; + --%test(Compare object type join by multi key) + procedure comp_obj_typ_col_jb_multi; - --%test(Compare nested table type join by missing nested key) - procedure comp_nest_tab_col_jb_nokey; + --%test(Compare object type join by missing nested key) + procedure comp_obj_typ_col_jb_nokey; + + --%test(Compare table type join by) + procedure compare_nest_tab_col_jb; + + --%test(Compare table type join by - Failure) + procedure compare_nest_tab_col_jb_fail; + + --%test(Compare table type join by mulitple columns) + procedure compare_nest_tab_cols_jb; + + --%test(Compare table type join by multiple columns- Failure) + procedure compare_nest_tab_cols_jb_fail; end; / From a7f167c93dc46b84b22d54bc5c54cec41cb46caf Mon Sep 17 00:00:00 2001 From: lwasylow Date: Tue, 22 May 2018 21:09:25 +0100 Subject: [PATCH 19/24] Update to handle new line in xmltype that caused pk value being returned with new lines char in 11g --- source/expectations/data_values/ut_compound_data_helper.pkb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index a89258c19..ef96116d9 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -160,7 +160,7 @@ create or replace package body ut_compound_data_helper is function get_pk_value (a_join_by_xpath varchar2,a_item_data xmltype) return clob is l_pk_value clob; begin - select extract(a_item_data,a_join_by_xpath).getclobval() into l_pk_value from dual; + select replace((extract(a_item_data,a_join_by_xpath).getclobval()),chr(10)) into l_pk_value from dual; return l_pk_value; exception when no_data_found then return 'null'; From b457d69dc38ea310c4dbd2743f4c129857801875 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Wed, 23 May 2018 13:11:19 +0100 Subject: [PATCH 20/24] Updates to the ordering of sets to get consistent results. --- .../data_values/ut_compound_data_helper.pkb | 15 +++++- .../test_expectations_cursor.pkb | 46 +++++++++---------- 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index ef96116d9..0e8eba809 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -186,7 +186,14 @@ create or replace package body ut_compound_data_helper is with diff_info as (select item_hash,pk_hash,duplicate_no from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) select rn,diff_type,diffed_row,pk_value from ( - select diff_type,diffed_row, dense_rank() over (order by pk_hash) rn,pk_value from + select + diff_type, diffed_row, + dense_rank() over (order by case when diff_type in ('Extra','Missing') then diff_type end, + case when diff_type in ('Actual','Expected') then pk_hash end, + case when diff_type in ('Extra','Missing') then pk_hash end, + case when diff_type in ('Actual','Expected') then diff_type end) rn, + pk_value, pk_hash + from ( select diff_type,diffed_row,pk_hash,pk_value from (select diff_type,data_item diffed_row,pk_hash,pk_value @@ -264,6 +271,7 @@ create or replace package body ut_compound_data_helper is where exp.pk_hash is null or act.pk_hash is null ) ) where rn <= :max_rows + order by rn, pk_hash, diff_type ]' bulk collect into l_results using a_diff_id, @@ -414,7 +422,10 @@ create or replace package body ut_compound_data_helper is ) act on exp.row_hash = act.row_hash and exp.duplicate_no = act.duplicate_no - where exp.row_hash is null or act.row_hash is null ) where rownum < :max_rows ]' + where exp.row_hash is null or act.row_hash is null + order by diffed_type, coalesce(exp.row_hash,act.row_hash), duplicate_no + ) + where rownum < :max_rows ]' bulk collect into l_results using a_diff_id, a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pkb b/test/core/expectations/compound_data/test_expectations_cursor.pkb index 7a87aab87..130f55380 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pkb +++ b/test/core/expectations/compound_data/test_expectations_cursor.pkb @@ -1073,10 +1073,10 @@ Rows: [ 2 differences ]% --Act ut3.ut.expect(l_actual).to_equal(l_expected).unordered; l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 2 ]% -Diff:% -Rows: [ 2 differences ] -Missing: test-667% -Extra: test-666%]'; +%Diff:% +%Rows: [ 2 differences ]% +%Extra: test-666% +%Missing: test-667%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1284,8 +1284,8 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = % ] was expected to equal: refcursor [ count = % ] %Diff:% %Rows: [ 1 differences ]% -%PK TEST - Expected:%-600% -%PK TEST - Actual:%-610%]'; +%PK TEST - Actual:%-610% +%PK TEST - Expected:%-600%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1310,8 +1310,8 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = % ] was expected to equal: refcursor [ count = % ] %Diff:% %Rows: [ 2 differences ]% -%PK TEST-600 - Missing % -%PK TEST-610 - Extra %]'; +%PK TEST-610 - Extra% +%PK TEST-600 - Missing%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1336,8 +1336,8 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = % ] was expected to equal: refcursor [ count = % ] %Diff:% %Rows: [ 1 differences ]% -%PK TESTY - Expected:%-600% -%PK TESTY - Actual:%-610%]'; +%PK TESTY - Actual:%-610% +%PK TESTY - Expected:%-600%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1493,11 +1493,11 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 3 ]% Diff:% Rows: [ 5 differences ] -Missing: 1Somethings 11% -Missing: 2Somethings 22% -Missing: 3Somethings 33% -Extra: 1Something 11% -Extra: 2Something 22%]'; +%Extra: 2Something 22% +%Extra: 1Something 11% +%Missing: 1Somethings 11% +%Missing: 2Somethings 22% +%Missing: 3Somethings 33%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1621,10 +1621,10 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 2 ]% %Diff:% %Rows: [ 2 differences ]% -%PK 1 - Expected: Somethings 1% -%PK 1 - Actual: Something 1% -%PK 2 - Expected: Somethings 2% -%PK 2 - Actual: Something 2%]'; +%PK % - Actual: %% +%PK % - Expected: %% +%PK % - Actual: %% +%PK % - Expected: %%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1688,10 +1688,10 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 2 ]% %Diff:% %Rows: [ 4 differences ]% -%PK 1Somethings 1 - Missing% -%PK 2Somethings 2 - Missing% -%PK 2Something 2 - Extra% -%PK 1Something 1 - Extra%]'; +%PK %% - Extra% +%PK %% - Extra% +%PK %% - Missing% +%PK %% - Missing%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); From 7577a952efe0409a96e816a2d82cae8f191efa8a Mon Sep 17 00:00:00 2001 From: lwasylow Date: Wed, 23 May 2018 19:28:00 +0100 Subject: [PATCH 21/24] Revert "Updates to the ordering of sets to get consistent results." This reverts commit b457d69dc38ea310c4dbd2743f4c129857801875. --- .../data_values/ut_compound_data_helper.pkb | 15 +----- .../test_expectations_cursor.pkb | 46 +++++++++---------- 2 files changed, 25 insertions(+), 36 deletions(-) diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index 0e8eba809..ef96116d9 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -186,14 +186,7 @@ create or replace package body ut_compound_data_helper is with diff_info as (select item_hash,pk_hash,duplicate_no from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) select rn,diff_type,diffed_row,pk_value from ( - select - diff_type, diffed_row, - dense_rank() over (order by case when diff_type in ('Extra','Missing') then diff_type end, - case when diff_type in ('Actual','Expected') then pk_hash end, - case when diff_type in ('Extra','Missing') then pk_hash end, - case when diff_type in ('Actual','Expected') then diff_type end) rn, - pk_value, pk_hash - from + select diff_type,diffed_row, dense_rank() over (order by pk_hash) rn,pk_value from ( select diff_type,diffed_row,pk_hash,pk_value from (select diff_type,data_item diffed_row,pk_hash,pk_value @@ -271,7 +264,6 @@ create or replace package body ut_compound_data_helper is where exp.pk_hash is null or act.pk_hash is null ) ) where rn <= :max_rows - order by rn, pk_hash, diff_type ]' bulk collect into l_results using a_diff_id, @@ -422,10 +414,7 @@ create or replace package body ut_compound_data_helper is ) act on exp.row_hash = act.row_hash and exp.duplicate_no = act.duplicate_no - where exp.row_hash is null or act.row_hash is null - order by diffed_type, coalesce(exp.row_hash,act.row_hash), duplicate_no - ) - where rownum < :max_rows ]' + where exp.row_hash is null or act.row_hash is null ) where rownum < :max_rows ]' bulk collect into l_results using a_diff_id, a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pkb b/test/core/expectations/compound_data/test_expectations_cursor.pkb index 130f55380..7a87aab87 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pkb +++ b/test/core/expectations/compound_data/test_expectations_cursor.pkb @@ -1073,10 +1073,10 @@ Rows: [ 2 differences ]% --Act ut3.ut.expect(l_actual).to_equal(l_expected).unordered; l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 2 ]% -%Diff:% -%Rows: [ 2 differences ]% -%Extra: test-666% -%Missing: test-667%]'; +Diff:% +Rows: [ 2 differences ] +Missing: test-667% +Extra: test-666%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1284,8 +1284,8 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = % ] was expected to equal: refcursor [ count = % ] %Diff:% %Rows: [ 1 differences ]% -%PK TEST - Actual:%-610% -%PK TEST - Expected:%-600%]'; +%PK TEST - Expected:%-600% +%PK TEST - Actual:%-610%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1310,8 +1310,8 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = % ] was expected to equal: refcursor [ count = % ] %Diff:% %Rows: [ 2 differences ]% -%PK TEST-610 - Extra% -%PK TEST-600 - Missing%]'; +%PK TEST-600 - Missing % +%PK TEST-610 - Extra %]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1336,8 +1336,8 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = % ] was expected to equal: refcursor [ count = % ] %Diff:% %Rows: [ 1 differences ]% -%PK TESTY - Actual:%-610% -%PK TESTY - Expected:%-600%]'; +%PK TESTY - Expected:%-600% +%PK TESTY - Actual:%-610%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1493,11 +1493,11 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 3 ]% Diff:% Rows: [ 5 differences ] -%Extra: 2Something 22% -%Extra: 1Something 11% -%Missing: 1Somethings 11% -%Missing: 2Somethings 22% -%Missing: 3Somethings 33%]'; +Missing: 1Somethings 11% +Missing: 2Somethings 22% +Missing: 3Somethings 33% +Extra: 1Something 11% +Extra: 2Something 22%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1621,10 +1621,10 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 2 ]% %Diff:% %Rows: [ 2 differences ]% -%PK % - Actual: %% -%PK % - Expected: %% -%PK % - Actual: %% -%PK % - Expected: %%]'; +%PK 1 - Expected: Somethings 1% +%PK 1 - Actual: Something 1% +%PK 2 - Expected: Somethings 2% +%PK 2 - Actual: Something 2%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1688,10 +1688,10 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 2 ]% %Diff:% %Rows: [ 4 differences ]% -%PK %% - Extra% -%PK %% - Extra% -%PK %% - Missing% -%PK %% - Missing%]'; +%PK 1Somethings 1 - Missing% +%PK 2Somethings 2 - Missing% +%PK 2Something 2 - Extra% +%PK 1Something 1 - Extra%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); From 890c1d2ff4f95e6022c3c104dbc6db3b77a40b4e Mon Sep 17 00:00:00 2001 From: lwasylow Date: Wed, 23 May 2018 19:44:49 +0100 Subject: [PATCH 22/24] Refactoring to simplify code --- .../data_values/ut_data_value_refcursor.tpb | 49 +++++++------------ .../data_values/ut_data_value_refcursor.tps | 2 - source/expectations/matchers/ut_equal.tpb | 6 +-- 3 files changed, 20 insertions(+), 37 deletions(-) diff --git a/source/expectations/data_values/ut_data_value_refcursor.tpb b/source/expectations/data_values/ut_data_value_refcursor.tpb index fe8c1f443..5b8ab8463 100644 --- a/source/expectations/data_values/ut_data_value_refcursor.tpb +++ b/source/expectations/data_values/ut_data_value_refcursor.tpb @@ -210,29 +210,13 @@ create or replace type body ut_data_value_refcursor as return l_result_string; end; - overriding member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2) return integer is - l_result integer := 0; - l_other ut_data_value_refcursor; - begin - if not a_other is of (ut_data_value_refcursor) then - raise value_error; - end if; - - l_other := treat(a_other as ut_data_value_refcursor); - --if column names/types are not equal - build a diff of column names and types - if ut_compound_data_helper.columns_hash( self, a_exclude_xpath, a_include_xpath ) - != ut_compound_data_helper.columns_hash( l_other, a_exclude_xpath, a_include_xpath ) - then - l_result := 1; - end if; - l_result := l_result + (self as ut_compound_data_value).compare_implementation(a_other, a_exclude_xpath, a_include_xpath); - return l_result; - end; - overriding member function compare_implementation (a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean) return integer is l_result integer := 0; l_other ut_data_value_refcursor; - l_missing_pk ut_compound_data_helper.tt_missing_pk := ut_compound_data_helper.tt_missing_pk(); + function is_pk_missing (a_pk_missing_tab ut_compound_data_helper.tt_missing_pk) return integer is + begin + return case when a_pk_missing_tab.count > 0 then 1 else 0 end; + end; begin if not a_other is of (ut_data_value_refcursor) then raise value_error; @@ -240,21 +224,26 @@ create or replace type body ut_data_value_refcursor as l_other := treat(a_other as ut_data_value_refcursor); - l_missing_pk := ut_compound_data_helper.is_pk_exists(self.key_info, l_other.key_info, a_exclude_xpath, a_include_xpath,a_join_by_xpath); --if we join by key and key is missing fail and report error - if a_join_by_xpath is not null and l_missing_pk.count > 0 then - l_result := 1; - return l_result; + if a_join_by_xpath is not null then + l_result := is_pk_missing(ut_compound_data_helper.is_pk_exists(self.key_info, l_other.key_info, a_exclude_xpath, a_include_xpath,a_join_by_xpath)); end if; - --if column names/types are not equal - build a diff of column names and types - if ut_compound_data_helper.columns_hash( self, a_exclude_xpath, a_include_xpath ) - != ut_compound_data_helper.columns_hash( l_other, a_exclude_xpath, a_include_xpath ) - then - l_result := 1; + if l_result = 0 then + --if column names/types are not equal - build a diff of column names and types + if ut_compound_data_helper.columns_hash( self, a_exclude_xpath, a_include_xpath ) + != ut_compound_data_helper.columns_hash( l_other, a_exclude_xpath, a_include_xpath ) + then + l_result := 1; + end if; + + if a_unordered then + l_result := l_result + (self as ut_compound_data_value).compare_implementation(a_other, a_exclude_xpath, a_include_xpath, a_join_by_xpath, a_unordered); + else + l_result := l_result + (self as ut_compound_data_value).compare_implementation(a_other, a_exclude_xpath, a_include_xpath); + end if; end if; - l_result := l_result + (self as ut_compound_data_value).compare_implementation(a_other, a_exclude_xpath, a_include_xpath, a_join_by_xpath, a_unordered); return l_result; end; diff --git a/source/expectations/data_values/ut_data_value_refcursor.tps b/source/expectations/data_values/ut_data_value_refcursor.tps index c73aaa538..4e175ce6b 100644 --- a/source/expectations/data_values/ut_data_value_refcursor.tps +++ b/source/expectations/data_values/ut_data_value_refcursor.tps @@ -39,8 +39,6 @@ create or replace type ut_data_value_refcursor under ut_compound_data_value( member procedure init(self in out nocopy ut_data_value_refcursor, a_value sys_refcursor), overriding member function to_string return varchar2, overriding member function diff( a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean := false ) return varchar2, - overriding member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2) return integer, overriding member function compare_implementation(a_other ut_data_value, a_exclude_xpath varchar2, a_include_xpath varchar2, a_join_by_xpath varchar2, a_unordered boolean) return integer - ) / diff --git a/source/expectations/matchers/ut_equal.tpb b/source/expectations/matchers/ut_equal.tpb index f9e6baaeb..3a163e9c9 100644 --- a/source/expectations/matchers/ut_equal.tpb +++ b/source/expectations/matchers/ut_equal.tpb @@ -233,11 +233,7 @@ create or replace type body ut_equal as if self.expected is of (ut_data_value_anydata) then l_result := 0 = treat(self.expected as ut_data_value_anydata).compare_implementation(a_actual, get_exclude_xpath(), get_include_xpath()); elsif self.expected is of (ut_data_value_refcursor) then - if get_unordered then - l_result := 0 = treat(self.expected as ut_data_value_refcursor).compare_implementation(a_actual, get_exclude_xpath(), get_include_xpath(), get_join_by_xpath(), get_unordered()); - else - l_result := 0 = treat(self.expected as ut_data_value_refcursor).compare_implementation(a_actual, get_exclude_xpath(), get_include_xpath()); - end if; + l_result := 0 = treat(self.expected as ut_data_value_refcursor).compare_implementation(a_actual, get_exclude_xpath(), get_include_xpath(), get_join_by_xpath(), get_unordered()); else l_result := equal_with_nulls((self.expected = a_actual), a_actual); end if; From 4d8b964a9df6ca1488219d59989ea9106bf6f708 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Wed, 23 May 2018 19:55:08 +0100 Subject: [PATCH 23/24] Revert "Revert "Updates to the ordering of sets to get consistent results."" This reverts commit 7577a952efe0409a96e816a2d82cae8f191efa8a. --- .../data_values/ut_compound_data_helper.pkb | 15 +++++- .../test_expectations_cursor.pkb | 46 +++++++++---------- 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb index ef96116d9..0e8eba809 100644 --- a/source/expectations/data_values/ut_compound_data_helper.pkb +++ b/source/expectations/data_values/ut_compound_data_helper.pkb @@ -186,7 +186,14 @@ create or replace package body ut_compound_data_helper is with diff_info as (select item_hash,pk_hash,duplicate_no from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid) select rn,diff_type,diffed_row,pk_value from ( - select diff_type,diffed_row, dense_rank() over (order by pk_hash) rn,pk_value from + select + diff_type, diffed_row, + dense_rank() over (order by case when diff_type in ('Extra','Missing') then diff_type end, + case when diff_type in ('Actual','Expected') then pk_hash end, + case when diff_type in ('Extra','Missing') then pk_hash end, + case when diff_type in ('Actual','Expected') then diff_type end) rn, + pk_value, pk_hash + from ( select diff_type,diffed_row,pk_hash,pk_value from (select diff_type,data_item diffed_row,pk_hash,pk_value @@ -264,6 +271,7 @@ create or replace package body ut_compound_data_helper is where exp.pk_hash is null or act.pk_hash is null ) ) where rn <= :max_rows + order by rn, pk_hash, diff_type ]' bulk collect into l_results using a_diff_id, @@ -414,7 +422,10 @@ create or replace package body ut_compound_data_helper is ) act on exp.row_hash = act.row_hash and exp.duplicate_no = act.duplicate_no - where exp.row_hash is null or act.row_hash is null ) where rownum < :max_rows ]' + where exp.row_hash is null or act.row_hash is null + order by diffed_type, coalesce(exp.row_hash,act.row_hash), duplicate_no + ) + where rownum < :max_rows ]' bulk collect into l_results using a_diff_id, a_exclude_xpath, a_include_xpath, a_expected_dataset_guid, diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pkb b/test/core/expectations/compound_data/test_expectations_cursor.pkb index 7a87aab87..130f55380 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pkb +++ b/test/core/expectations/compound_data/test_expectations_cursor.pkb @@ -1073,10 +1073,10 @@ Rows: [ 2 differences ]% --Act ut3.ut.expect(l_actual).to_equal(l_expected).unordered; l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 2 ]% -Diff:% -Rows: [ 2 differences ] -Missing: test-667% -Extra: test-666%]'; +%Diff:% +%Rows: [ 2 differences ]% +%Extra: test-666% +%Missing: test-667%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1284,8 +1284,8 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = % ] was expected to equal: refcursor [ count = % ] %Diff:% %Rows: [ 1 differences ]% -%PK TEST - Expected:%-600% -%PK TEST - Actual:%-610%]'; +%PK TEST - Actual:%-610% +%PK TEST - Expected:%-600%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1310,8 +1310,8 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = % ] was expected to equal: refcursor [ count = % ] %Diff:% %Rows: [ 2 differences ]% -%PK TEST-600 - Missing % -%PK TEST-610 - Extra %]'; +%PK TEST-610 - Extra% +%PK TEST-600 - Missing%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1336,8 +1336,8 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = % ] was expected to equal: refcursor [ count = % ] %Diff:% %Rows: [ 1 differences ]% -%PK TESTY - Expected:%-600% -%PK TESTY - Actual:%-610%]'; +%PK TESTY - Actual:%-610% +%PK TESTY - Expected:%-600%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1493,11 +1493,11 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 3 ]% Diff:% Rows: [ 5 differences ] -Missing: 1Somethings 11% -Missing: 2Somethings 22% -Missing: 3Somethings 33% -Extra: 1Something 11% -Extra: 2Something 22%]'; +%Extra: 2Something 22% +%Extra: 1Something 11% +%Missing: 1Somethings 11% +%Missing: 2Somethings 22% +%Missing: 3Somethings 33%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1621,10 +1621,10 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 2 ]% %Diff:% %Rows: [ 2 differences ]% -%PK 1 - Expected: Somethings 1% -%PK 1 - Actual: Something 1% -%PK 2 - Expected: Somethings 2% -%PK 2 - Actual: Something 2%]'; +%PK % - Actual: %% +%PK % - Expected: %% +%PK % - Actual: %% +%PK % - Expected: %%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); @@ -1688,10 +1688,10 @@ Diff:% l_expected_message := q'[%Actual: refcursor [ count = 2 ] was expected to equal: refcursor [ count = 2 ]% %Diff:% %Rows: [ 4 differences ]% -%PK 1Somethings 1 - Missing% -%PK 2Somethings 2 - Missing% -%PK 2Something 2 - Extra% -%PK 1Something 1 - Extra%]'; +%PK %% - Extra% +%PK %% - Extra% +%PK %% - Missing% +%PK %% - Missing%]'; l_actual_message := ut3.ut_expectation_processor.get_failed_expectations()(1).message; --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); From f77f13abdeb33f7c2ad1e3d77cb0f269a292f0c2 Mon Sep 17 00:00:00 2001 From: lwasylow Date: Thu, 24 May 2018 20:33:19 +0100 Subject: [PATCH 24/24] Update to test and docs --- docs/userguide/advanced_data_comparison.md | 2 ++ .../test_expectations_cursor.pkb | 35 +++++++++++++++++++ .../test_expectations_cursor.pks | 3 ++ 3 files changed, 40 insertions(+) diff --git a/docs/userguide/advanced_data_comparison.md b/docs/userguide/advanced_data_comparison.md index cc488da11..c776f8596 100644 --- a/docs/userguide/advanced_data_comparison.md +++ b/docs/userguide/advanced_data_comparison.md @@ -143,6 +143,8 @@ You can now join two cursors by defining a primary key or composite key that wil Join by option can be used in conjunction with include or exclude options. However if any of the join keys is part of exclude set, comparison will fail and report to user that sets could not be joined on specific key (excluded). +Join by options currently doesn't support nested table inside cursor. + ```sql procedure join_by_username is l_actual sys_refcursor; diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pkb b/test/core/expectations/compound_data/test_expectations_cursor.pkb index 130f55380..efb1ca845 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pkb +++ b/test/core/expectations/compound_data/test_expectations_cursor.pkb @@ -1696,5 +1696,40 @@ Diff:% --Assert ut.expect(l_actual_message).to_be_like(l_expected_message); end; + + procedure comparet_tabtype_as_cols_jb is + l_actual sys_refcursor; + l_expected sys_refcursor; + l_actual_tab ut3.ut_key_value_pairs := ut3.ut_key_value_pairs(); + l_expected_tab ut3.ut_key_value_pairs := ut3.ut_key_value_pairs(); + l_expected_message varchar2(32767); + l_actual_message varchar2(32767); + begin + select ut3.ut_key_value_pair(rownum,'Something '||rownum) + bulk collect into l_actual_tab + from dual connect by level <=2; + + select ut3.ut_key_value_pair(rownum,'Somethings '||rownum) + bulk collect into l_expected_tab + from dual connect by level <=2; + + --Arrange + open l_actual for select rownum rn, l_actual_tab + from dual connect by level <=2; + + open l_expected for select key,value + from table(l_expected_tab) order by 1 desc; + + --Act + ut3.ut.expect(l_actual).to_equal(l_expected).join_by(ut3.ut_varchar2_list('RN,L_ACTUAL_TAB/KEY')); + ut3.ut.fail('Expected Exception'); + exception + when others then + l_expected_message := q'[%PLS-00306: wrong number or types of arguments in call to 'CONVERTOBJECT'%]'; + l_actual_message := SQLERRM; + --Assert + ut.expect(l_actual_message).to_be_like(l_expected_message); + end; + end; / diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pks b/test/core/expectations/compound_data/test_expectations_cursor.pks index 185697cf8..bd8cf45a0 100644 --- a/test/core/expectations/compound_data/test_expectations_cursor.pks +++ b/test/core/expectations/compound_data/test_expectations_cursor.pks @@ -292,5 +292,8 @@ create or replace package test_expectations_cursor is --%test(Compare table type join by multiple columns- Failure) procedure compare_nest_tab_cols_jb_fail; + --%test(Compare table type as column join by multiple columns - PLS Failure) + procedure comparet_tabtype_as_cols_jb; + end; /