Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 784c509

Browse files
lwasylowlwasylow
committed
Update pk join cursor
fix to get columns diff
1 parent aaa0cc4 commit 784c509

6 files changed

Lines changed: 210 additions & 78 deletions

File tree

‎source/expectations/data_values/ut_compound_data_helper.pkb‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,32 @@ create or replace package body ut_compound_data_helper is
8686
end if;
8787
return l_filter;
8888
end;
89+
90+
/**
91+
* Current get column filter shaving off ROW tag during extract, this not working well with include and XMLTABLE option
92+
* so when there is extract we artificially inject removed tag
93+
**/
94+
function get_columns_row_filter(
95+
a_exclude_xpath varchar2, a_include_xpath varchar2,
96+
a_table_alias varchar2 := 'ucd', a_column_alias varchar2 := 'item_data'
97+
) return varchar2 is
98+
l_filter varchar2(32767);
99+
l_source_column varchar2(500) := a_table_alias||'.'||a_column_alias;
100+
begin
101+
-- this SQL statement is constructed in a way that we always get the same number and ordering of substitution variables
102+
-- That is, we always get: l_exclude_xpath, l_include_xpath
103+
-- regardless if the variables are NULL (not to be used) or NOT NULL and will be used for filtering
104+
if a_exclude_xpath is null and a_include_xpath is null then
105+
l_filter := ':l_exclude_xpath, :l_include_xpath, '||l_source_column||' as '||a_column_alias;
106+
elsif a_exclude_xpath is not null and a_include_xpath is null then
107+
l_filter := 'deletexml( '||l_source_column||', :l_exclude_xpath ) as '||a_column_alias||', :l_include_xpath';
108+
elsif a_exclude_xpath is null and a_include_xpath is not null then
109+
l_filter := ':l_exclude_xpath, xmlelement("ROW",extract( '||l_source_column||', :l_include_xpath )) as '||a_column_alias;
110+
elsif a_exclude_xpath is not null and a_include_xpath is not null then
111+
l_filter := 'xmlelement("ROW",extract( deletexml( '||l_source_column||', :l_exclude_xpath ), :l_include_xpath )) as '||a_column_alias;
112+
end if;
113+
return l_filter;
114+
end;
89115

90116
function get_columns_diff(
91117
a_expected xmltype, a_actual xmltype, a_exclude_xpath varchar2, a_include_xpath varchar2
@@ -94,7 +120,7 @@ create or replace package body ut_compound_data_helper is
94120
l_sql varchar2(32767);
95121
l_results tt_column_diffs;
96122
begin
97-
l_column_filter := get_columns_filter(a_exclude_xpath, a_include_xpath);
123+
l_column_filter := get_columns_row_filter(a_exclude_xpath, a_include_xpath);
98124
l_sql := q'[
99125
with
100126
expected_cols as ( select :a_expected as item_data from dual ),
@@ -157,6 +183,7 @@ create or replace package body ut_compound_data_helper is
157183
execute immediate l_sql
158184
bulk collect into l_results
159185
using a_expected, a_actual, a_exclude_xpath, a_include_xpath, a_exclude_xpath, a_include_xpath;
186+
160187
return l_results;
161188
end;
162189

@@ -349,8 +376,8 @@ create or replace package body ut_compound_data_helper is
349376
/**
350377
* Since its unordered search we cannot select max rows from diffs as we miss some comparision records
351378
* We will restrict output on higher level of select
352-
*/
353-
379+
*/
380+
354381
execute immediate q'[with
355382
diff_info as (select item_hash,duplicate_no from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid)
356383
select duplicate_no,

‎source/expectations/data_values/ut_compound_data_tmp.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ create global temporary table ut_compound_data_tmp(
1616
item_no integer,
1717
item_data xmltype,
1818
item_hash raw(128),
19+
pk_hash raw(128),
1920
duplicate_no integer,
2021
constraint ut_cmp_data_tmp_hash_pk unique (data_id,item_no, item_hash , duplicate_no)
2122
) on commit preserve rows;

‎source/expectations/data_values/ut_compound_data_value.tpb‎

Lines changed: 58 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -250,24 +250,32 @@ create or replace type body ut_compound_data_value as
250250
l_row_diffs ut_compound_data_helper.tt_row_diffs;
251251
c_max_rows constant integer := 20;
252252

253-
type t_pk_val_rec is record
254-
(
255-
data_id raw(32),
256-
item_hash raw(128),
257-
pk_hash raw(128),
258-
pk_value varchar2(4000)
259-
);
260-
261-
type t_pk_val_tab is table of t_pk_val_rec;
262-
l_pk_val_tab t_pk_val_tab;
263-
264-
function get_column_xpath(a_join_by_xpath varchar2) return varchar2 is
253+
function get_pk_value(a_join_by_xpath varchar2) return varchar2 is
265254
l_column varchar2(32767);
266255
begin
256+
/* due to possibility of key being to columns we cannot use xmlextractvalue
257+
usage of xmlagg is possible however it greatly complicates code and performance is impacted.
258+
xpath to be looked at or regex
259+
*/
267260
if a_join_by_xpath is not null then
268-
l_column := l_ut_owner ||'.ut_compound_data_helper.get_hash(extract(t.item_data,:join_by_xpath).GetClobVal()) pk_hash ';
261+
l_column := ' extract(t.item_data,:join_by_xpath).GetStringVal() pk_value ';
269262
else
270-
l_column := ':join_by_xpath pk_hash ';
263+
l_column := ' :join_by_xpath pk_value';
264+
end if;
265+
return l_column;
266+
end;
267+
268+
function get_column_pk_hash(a_join_by_xpath varchar2) return varchar2 is
269+
l_column varchar2(32767);
270+
begin
271+
/* due to possibility of key being to columns we cannot use xmlextractvalue
272+
usage of xmlagg is possible however it greatly complicates code and performance is impacted.
273+
xpath to be looked at or regex
274+
*/
275+
if a_join_by_xpath is not null then
276+
l_column := l_ut_owner ||'.ut_compound_data_helper.get_hash(extract(ucd.item_data,:join_by_xpath).GetClobVal()) pk_hash';
277+
else
278+
l_column := ':join_by_xpath pk_hash';
271279
end if;
272280
return l_column;
273281
end;
@@ -281,81 +289,58 @@ create or replace type body ut_compound_data_value as
281289

282290
l_diff_id := ut_compound_data_helper.get_hash(self.data_id||l_other.data_id);
283291
l_column_filter := ut_compound_data_helper.get_columns_filter(a_exclude_xpath, a_include_xpath);
284-
292+
285293
/**
286294
* Due to incompatibility issues in XML between 11 and 12.2 and 12.1 versions we will prepopulate pk_hash upfront to
287295
* avoid optimizer incorrectly rewrite and causing NULL error or ORA-600
288-
**/
289-
-- Pre generate hash minus to leave only onese that are diffrent, for example duplicates or diffrent hash
290-
/*if a_join_by_xpath is not null then
291-
execute immediate q'[select
292-
data_id,item_hash,pk_hash,
293-
listagg(extractvalue(tcd.column_value, '/*'), '; ') within GROUP(ORDER BY item_hash) pk_value
294-
from
295-
(select
296-
ucd.column_value row_data,
297-
]'|| l_ut_owner ||q'[.ut_compound_data_helper.get_hash(extract(ucd.column_value,:join_xpath).GetClobVal()) pk_hash ,
298-
t.item_hash,
299-
t.data_id
300-
from ]' || l_ut_owner || q'[.ut_compound_data_tmp t ,
301-
table(xmlsequence(extract(t.item_data,'/*'))) ucd
302-
where data_id = :self_guid or data_id = :other_guid ),
303-
table(xmlsequence(extract(row_data ,:join_xpath))) tcd
304-
group by pk_hash,item_hash,data_id]'
305-
bulk collect into l_pk_val_tab using a_join_by_xpath,self.data_id, l_other.data_id,a_join_by_xpath;
296+
**/
297+
execute immediate 'merge into ' || l_ut_owner || '.ut_compound_data_tmp tgt
298+
using (
299+
select '||l_ut_owner ||'.ut_compound_data_helper.get_hash(ucd.item_data.getclobval()) item_hash,
300+
pk_hash, ucd.item_no, ucd.data_id
301+
from
302+
(
303+
select '||l_column_filter||','||get_column_pk_hash(a_join_by_xpath)||', item_no, data_id
304+
from ' || l_ut_owner || q'[.ut_compound_data_tmp ucd
305+
where data_id = :self_guid or data_id = :other_guid
306+
) ucd
307+
) src
308+
on (tgt.item_no = src.item_no and tgt.data_id = src.data_id)
309+
when matched then update
310+
set tgt.item_hash = src.item_hash,
311+
tgt.pk_hash = src.pk_hash ]'
312+
using a_exclude_xpath, a_include_xpath,a_join_by_xpath,self.data_id, l_other.data_id;
306313

307-
forall pk_vals in 1..l_pk_val_tab.COUNT
308-
update ut_compound_data_tmp
309-
set pk_hash = l_pk_val_tab(pk_vals).pk_hash
310-
,pk_value = l_pk_val_tab(pk_vals).pk_value
311-
where data_id = l_pk_val_tab(pk_vals).data_id
312-
and item_hash = l_pk_val_tab(pk_vals).item_hash;
313-
end if;
314-
*/
315-
316-
execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no)
317-
with calc_pk as
318-
( select data_id,item_hash, '||get_column_xpath(a_join_by_xpath)||'
319-
from ' || l_ut_owner || '.ut_compound_data_tmp t
320-
)
321-
select distinct :diff_id,tmp.item_hash,tmp.pk_hash,tmp.duplicate_no
314+
/* Peform minus on two sets two get diffrences that will be used later on to print results */
315+
execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no,pk_value)
316+
with source_data as
317+
( select t.data_id,t.item_hash,row_number() over (partition by t.pk_hash,t.item_hash,t.data_id order by 1,2) duplicate_no,
318+
pk_hash, '||get_pk_value(a_join_by_xpath)||'
319+
from ' || l_ut_owner || '.ut_compound_data_tmp t
320+
where data_id = :self_guid or data_id = :other_guid
321+
)
322+
select distinct :diff_id,tmp.item_hash,tmp.pk_hash,tmp.duplicate_no,pk_value
322323
from(
323324
(
324-
select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no,
325-
pc.pk_hash
326-
from ' || l_ut_owner || '.ut_compound_data_tmp t,
327-
calc_pk pc
325+
select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value
326+
from source_data t
328327
where t.data_id = :self_guid
329-
and pc.data_id = t.data_id
330-
and pc.item_hash = t.item_hash
331328
minus
332-
select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no,
333-
pc.pk_hash
334-
from ' || l_ut_owner || '.ut_compound_data_tmp t,
335-
calc_pk pc
329+
select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value
330+
from source_data t
336331
where t.data_id = :other_guid
337-
and pc.data_id = t.data_id
338-
and pc.item_hash = t.item_hash
339332
)
340333
union all
341334
(
342-
select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no,
343-
pc.pk_hash
344-
from ' || l_ut_owner || '.ut_compound_data_tmp t,
345-
calc_pk pc
335+
select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value
336+
from source_data t
346337
where t.data_id = :other_guid
347-
and pc.data_id = t.data_id
348-
and pc.item_hash = t.item_hash
349338
minus
350-
select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no,
351-
pc.pk_hash
352-
from ' || l_ut_owner || '.ut_compound_data_tmp t,
353-
calc_pk pc
339+
select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value
340+
from source_data t
354341
where t.data_id = :self_guid
355-
and pc.data_id = t.data_id
356-
and pc.item_hash = t.item_hash
357342
))tmp'
358-
using a_join_by_xpath,
343+
using a_join_by_xpath,self.data_id, l_other.data_id,
359344
l_diff_id,
360345
self.data_id, l_other.data_id,
361346
l_other.data_id,self.data_id;

‎source/expectations/data_values/ut_data_value_refcursor.tpb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@ create or replace type body ut_data_value_refcursor as
6060
l_xml := dbms_xmlgen.getxmltype(l_ctx);
6161

6262
execute immediate
63-
'insert into ' || l_ut_owner || '.ut_compound_data_tmp(data_id, item_no, item_data, item_hash) ' ||
64-
'select :self_guid, :self_row_count + rownum, value(a),'||l_ut_owner||'.ut_compound_data_helper.get_hash(value(a).GetClobVal()) ' ||
63+
'insert into ' || l_ut_owner || '.ut_compound_data_tmp(data_id, item_no, item_data) ' ||
64+
'select :self_guid, :self_row_count + rownum, value(a) ' ||
6565
' from table( xmlsequence( extract(:l_xml,''ROWSET/*'') ) ) a'
6666
using in self.data_id, self.elements_count, l_xml;
6767

‎test/core/expectations/compound_data/test_expectations_cursor.pkb‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1127,5 +1127,103 @@ Extra: <ROW><USERNAME>test</USERNAME><USER_ID>-666</USER_ID></ROW>%]';
11271127
ut.expect(expectations.failed_expectations_data()).to_be_empty();
11281128
end;
11291129

1130+
procedure unord_incl_cols_as_list
1131+
as
1132+
l_actual sys_refcursor;
1133+
l_expected sys_refcursor;
1134+
begin
1135+
--Arrange
1136+
open l_actual for select rownum as rn, 'a' as "A_Column", 'c' as A_COLUMN, 'x' SOME_COL, 'd' "Some_Col" from dual a connect by level < 4;
1137+
open l_expected for select rownum as rn, 'a' as "A_Column", 'd' as A_COLUMN, 'x' SOME_COL, 'c' "Some_Col" from dual a connect by level < 4;
1138+
--Act
1139+
ut3.ut.expect(l_actual).to_equal(l_expected).include(ut3.ut_varchar2_list('RN','//A_Column','SOME_COL')).unordered;
1140+
--Assert
1141+
ut.expect(expectations.failed_expectations_data()).to_be_empty();
1142+
end;
1143+
1144+
procedure joinby_incl_cols_as_list
1145+
as
1146+
l_actual sys_refcursor;
1147+
l_expected sys_refcursor;
1148+
begin
1149+
--Arrange
1150+
open l_actual for select rownum as rn, 'a' as "A_Column", 'c' as A_COLUMN, 'x' SOME_COL, 'd' "Some_Col" from dual a connect by level < 4;
1151+
open l_expected for select rownum as rn, 'a' as "A_Column", 'd' as A_COLUMN, 'x' SOME_COL, 'c' "Some_Col" from dual a connect by level < 4;
1152+
--Act
1153+
ut3.ut.expect(l_actual).to_equal(l_expected).include(ut3.ut_varchar2_list('RN','//A_Column','SOME_COL')).join_by('SOME_COL');
1154+
--Assert
1155+
ut.expect(expectations.failed_expectations_data()).to_be_empty();
1156+
end;
1157+
1158+
procedure joinby_excl_cols_as_list
1159+
as
1160+
l_actual sys_refcursor;
1161+
l_expected sys_refcursor;
1162+
begin
1163+
--Arrange
1164+
open l_actual for select rownum as rn, 'a' as "A_Column", 'c' as A_COLUMN, 'x' SOME_COL, 'd' "Some_Col" from dual a connect by level < 4;
1165+
open l_expected for select rownum as rn, 'a' as "A_Column", 'd' as A_COLUMN, 'x' SOME_COL, 'c' "Some_Col" from dual a connect by level < 4;
1166+
--Act
1167+
ut3.ut.expect(l_actual).to_equal(l_expected).exclude(ut3.ut_varchar2_list('//Some_Col','A_COLUMN')).join_by('SOME_COL');
1168+
--Assert
1169+
ut.expect(expectations.failed_expectations_data()).to_be_empty();
1170+
end;
1171+
1172+
procedure unord_excl_cols_as_list
1173+
as
1174+
l_actual sys_refcursor;
1175+
l_expected sys_refcursor;
1176+
begin
1177+
--Arrange
1178+
open l_actual for select rownum as rn, 'a' as "A_Column", 'c' as A_COLUMN, 'x' SOME_COL, 'd' "Some_Col" from dual a connect by level < 4;
1179+
open l_expected for select rownum as rn, 'a' as "A_Column", 'd' as A_COLUMN, 'x' SOME_COL, 'c' "Some_Col" from dual a connect by level < 4;
1180+
--Act
1181+
ut3.ut.expect(l_actual).to_equal(l_expected).exclude(ut3.ut_varchar2_list('A_COLUMN|//Some_Col')).unordered;
1182+
--Assert
1183+
ut.expect(expectations.failed_expectations_data()).to_be_empty();
1184+
end;
1185+
1186+
procedure excl_dif_cols_as_list
1187+
as
1188+
l_actual sys_refcursor;
1189+
l_expected sys_refcursor;
1190+
begin
1191+
--Arrange
1192+
open l_actual for select rownum as rn, 'TEST' as A_COLUMN from dual a connect by level < 4;
1193+
open l_expected for select rownum as rn, 1 as A_COLUMN from dual a connect by level < 4;
1194+
--Act
1195+
ut3.ut.expect(l_actual).to_equal(l_expected).exclude(ut3.ut_varchar2_list('A_COLUMN'));
1196+
--Assert
1197+
ut.expect(expectations.failed_expectations_data()).to_be_empty();
1198+
end;
1199+
1200+
procedure inlc_dif_cols_as_list
1201+
as
1202+
l_actual sys_refcursor;
1203+
l_expected sys_refcursor;
1204+
begin
1205+
--Arrange
1206+
open l_actual for select rownum as rn, 'TEST' as A_COLUMN from dual a connect by level < 4;
1207+
open l_expected for select rownum as rn, 1 as A_COLUMN from dual a connect by level < 4;
1208+
--Act
1209+
ut3.ut.expect(l_actual).to_equal(l_expected).include(ut3.ut_varchar2_list('RN'));
1210+
--Assert
1211+
ut.expect(expectations.failed_expectations_data()).to_be_empty();
1212+
end;
1213+
1214+
procedure inlc_exc_dif_cols_as_list
1215+
as
1216+
l_actual sys_refcursor;
1217+
l_expected sys_refcursor;
1218+
begin
1219+
--Arrange
1220+
open l_actual for select rownum as rn, 'TEST' as A_COLUMN from dual a connect by level < 4;
1221+
open l_expected for select rownum as rn, 1 as A_COLUMN from dual a connect by level < 4;
1222+
--Act
1223+
ut3.ut.expect(l_actual).to_equal(l_expected).include(ut3.ut_varchar2_list('RN')).exclude(ut3.ut_varchar2_list('A_COLUMN'));
1224+
--Assert
1225+
ut.expect(expectations.failed_expectations_data()).to_be_empty();
1226+
end;
1227+
11301228
end;
11311229
/

‎test/core/expectations/compound_data/test_expectations_cursor.pks‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,5 +214,26 @@ create or replace package test_expectations_cursor is
214214
--%test( Compare cursors join by single key more than 1000 rows)
215215
procedure cursor_joinby_compare_1000;
216216

217+
--%test(Unordered List of columns to include)
218+
procedure unord_incl_cols_as_list;
219+
220+
--%test(Join By List of columns to include)
221+
procedure joinby_incl_cols_as_list;
222+
223+
--%test(Unordered List of columns to exclude)
224+
procedure unord_excl_cols_as_list;
225+
226+
--%test(Join By List of columns to exclude)
227+
procedure joinby_excl_cols_as_list;
228+
229+
--%test(Exclude columns of diffrent type)
230+
procedure excl_dif_cols_as_list;
231+
232+
--%test(Include column of same type leaving diffrent type out)
233+
procedure inlc_dif_cols_as_list;
234+
235+
--%test(Include column of same type leaving diffrent type out and exclude diffrent type)
236+
procedure inlc_exc_dif_cols_as_list;
237+
217238
end;
218239
/

0 commit comments

Comments
 (0)