Update to sort random order cause of optimizer

lwasylow · lwasylow · commit aaa0cc465eb9 · 2018-05-07T22:19:01.000+01:00
Capture PK Hash to improve lookup
diff --git a/docs/userguide/advanced_data_comparison.md b/docs/userguide/advanced_data_comparison.md
@@ -137,7 +137,7 @@ Above test will result in two differences of one row extra and one row missing.
 
 
 
-## Join by option
+## Join By option
 
 You can now join two cursors by defining a primary key or composite key that will be used to uniquely identify and compare rows. This option allows us to exactly show which rows are missing, extra and which are different without ordering clause.
 
@@ -157,7 +157,15 @@ begin
     ut.expect( l_actual ).to_equal( l_expected ).join_by('USERNAME');
 end;
 ```
-This will show you diffrence in row 'TEST' regardless of order.
+This will show you difference in row 'TEST' regardless of order.
+
+```sql
+      Rows: [ 1 differences ]
+        Expected: <USER_ID>-600</USER_ID> for key: TEST
+        Actual:   <USER_ID>-610</USER_ID> for key: TEST
+```
+
+**Please note that .join_by option will take longer to process due to need of parsing via primary keys.**
 
 ## Defining item as XPath
 When using XPath expression, keep in mind the following:
diff --git a/source/expectations/data_values/ut_compound_data_diff_tmp.sql b/source/expectations/data_values/ut_compound_data_diff_tmp.sql
@@ -15,7 +15,8 @@ create global temporary table ut_compound_data_diff_tmp(
   diff_id    raw(128),
   item_no    integer,
   pk_hash    raw(128),
-  item_hash   raw(128),
+  pk_value   varchar2(4000),
+  item_hash  raw(128),
   duplicate_no integer,
   constraint ut_compound_data_diff_tmp_uk1 unique (diff_id,duplicate_no,item_no,item_hash, pk_hash),
   constraint ut_compound_data_diff_tmp_chk check(
diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb
@@ -176,15 +176,15 @@ create or replace package body ut_compound_data_helper is
     **/
     
     execute immediate q'[
-    with diff_info as (select item_hash,pk_hash from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid)
-      select rn,diff_type,diffed_row from
+    with diff_info as (select item_hash,pk_hash,pk_value from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid)
+      select rn,diff_type,diffed_row,pk_value from
       (
-      select diff_type,diffed_row, dense_rank() over (order by pk_hash) rn from
+      select diff_type,diffed_row, dense_rank() over (order by pk_hash) rn,pk_value from
       (
-      select diff_type,diffed_row,pk_hash from
-        (select diff_type,data_item diffed_row,pk_hash
+      select diff_type,diffed_row,pk_hash,pk_value from
+        (select diff_type,data_item diffed_row,pk_hash,pk_value
          from
-          (select nvl(exp.pk_hash, act.pk_hash) pk_hash,
+          (select nvl(exp.pk_hash, act.pk_hash) pk_hash,nvl(exp.pk_value, act.pk_value) pk_value,
              xmlserialize(content exp.row_data no indent)  exp_item,
              xmlserialize(content act.row_data no indent)  act_item
              from 
@@ -193,10 +193,11 @@ create or replace package body ut_compound_data_helper is
                 (select ucd.column_value row_data,
                  r.item_hash row_hash,
                  r.pk_hash ,
+                 r.pk_value,
                  ucd.column_value.getRootElement() col_name,
                  ucd.column_value.getclobval() col_val
                  from 
-                  (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash
+                  (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash, i.pk_value
                    from ut_compound_data_tmp ucd,
                    diff_info i
                    where ucd.data_id = :self_guid
@@ -211,10 +212,11 @@ create or replace package body ut_compound_data_helper is
                  (select ucd.column_value row_data,
                   r.item_hash row_hash,
                   r.pk_hash ,
+                  r.pk_value,
                   ucd.column_value.getRootElement() col_name,
                   ucd.column_value.getclobval() col_val
                   from 
-                   (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash
+                   (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash, i.pk_value
                     from ut_compound_data_tmp ucd,
                     diff_info i
                     where ucd.data_id = :other_guid
@@ -231,7 +233,8 @@ create or replace package body ut_compound_data_helper is
       union all       
       select case when exp.pk_hash is null then 'Extra:' else 'Missing:' end as diff_type,
              xmlserialize(content nvl(exp.item_data, act.item_data) no indent) diffed_row,
-             coalesce(exp.pk_hash,act.pk_hash) pk_hash
+             coalesce(exp.pk_hash,act.pk_hash) pk_hash,
+             null pk_value
         from (select extract(ucd.item_data,'/*/*') item_data,i.pk_hash
                 from ut_compound_data_tmp ucd,
                 diff_info i
@@ -269,9 +272,12 @@ create or replace package body ut_compound_data_helper is
     l_column_filter := get_columns_filter(a_exclude_xpath,a_include_xpath);
     execute immediate q'[
       with
-        diff_info as (select item_no from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid and rownum <= :max_rows)
+        diff_info as ( select item_no 
+                       from 
+                         (select item_no from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid order by item_no asc) 
+                       where rownum <= :max_rows)
       select *
-        from (select rn, diff_type, xmlserialize(content data_item no indent) diffed_row
+        from (select rn, diff_type, xmlserialize(content data_item no indent) diffed_row, null pk_value
                 from (select nvl(exp.rn, act.rn) rn,
                              xmlagg(exp.col order by exp.col_no) exp_item,
                              xmlagg(act.col order by act.col_no) act_item
@@ -307,7 +313,8 @@ create or replace package body ut_compound_data_helper is
       union all
       select nvl(exp.item_no, act.item_no) rn,
              case when exp.item_no is null then 'Extra:' else 'Missing:' end as diff_type,
-             xmlserialize(content nvl(exp.item_data, act.item_data) no indent) diffed_row
+             xmlserialize(content nvl(exp.item_data, act.item_data) no indent) diffed_row,
+             null pk_value
         from (select ucd.item_no, extract(ucd.item_data,'/*/*') item_data
                 from ut_compound_data_tmp ucd
                where ucd.data_id = :self_guid
@@ -348,7 +355,8 @@ create or replace package body ut_compound_data_helper is
       diff_info as (select item_hash,duplicate_no from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid)
       select duplicate_no,
              diffed_type,
-             diffed_row
+             diffed_row,
+             null pk_value
       from
       (select  
         coalesce(exp.duplicate_no,act.duplicate_no) duplicate_no,
diff --git a/source/expectations/data_values/ut_compound_data_helper.pks b/source/expectations/data_values/ut_compound_data_helper.pks
@@ -31,7 +31,8 @@ create or replace package ut_compound_data_helper authid definer is
   type t_row_diffs is record(
     rn            integer,
     diff_type     varchar2(250),
-    diffed_row    clob
+    diffed_row    clob,
+    pk_value      varchar2(4000)
   );
 
   type tt_row_diffs is table of t_row_diffs;
diff --git a/source/expectations/data_values/ut_compound_data_tmp.sql b/source/expectations/data_values/ut_compound_data_tmp.sql
@@ -16,7 +16,6 @@ create global temporary table ut_compound_data_tmp(
   item_no          integer,
   item_data        xmltype,
   item_hash        raw(128),
-  pk_hash          raw(128),
   duplicate_no     integer,
   constraint ut_cmp_data_tmp_hash_pk unique (data_id,item_no, item_hash , duplicate_no)
 ) on commit preserve rows;
diff --git a/source/expectations/data_values/ut_compound_data_value.tpb b/source/expectations/data_values/ut_compound_data_value.tpb
@@ -139,6 +139,7 @@ create or replace type body ut_compound_data_value as
     l_actual            ut_compound_data_value;
     l_diff_id           ut_compound_data_helper.t_hash;
     l_row_diffs         ut_compound_data_helper.tt_row_diffs;
+     
   begin
     if not a_other is of (ut_compound_data_value) then
       raise value_error;
@@ -173,7 +174,11 @@ create or replace type body ut_compound_data_value as
       ut_utils.append_to_clob( l_result, l_message );
       for i in 1 .. l_row_diffs.count loop
         l_results.extend;
-        l_results(l_results.last) := '  Row No. '||l_row_diffs(i).rn||' - '||rpad(l_row_diffs(i).diff_type,10)||l_row_diffs(i).diffed_row;
+        if a_join_by_xpath is not null and l_row_diffs(i).pk_value is not null then
+          l_results(l_results.last) := '  '||rpad(l_row_diffs(i).diff_type,10)||l_row_diffs(i).diffed_row||' for key: '||l_row_diffs(i).pk_value;
+        else
+          l_results(l_results.last) := '  Row No. '||l_row_diffs(i).rn||' - '||rpad(l_row_diffs(i).diff_type,10)||l_row_diffs(i).diffed_row;
+        end if;
       end loop;
       ut_utils.append_to_clob(l_result,l_results);
     end if;
@@ -245,16 +250,28 @@ create or replace type body ut_compound_data_value as
     l_row_diffs       ut_compound_data_helper.tt_row_diffs;
     c_max_rows        constant integer := 20;
     
-    type t_fetch_pk_hash_rec is record
+    type t_pk_val_rec is record
     (
-    data_id          raw(32),
-    item_hash        raw(128),
-    item_data        xmltype
+    data_id   raw(32),
+    item_hash raw(128),
+    pk_hash   raw(128),
+    pk_value  varchar2(4000)    
     );
-    type t_fetch_pk_hash_tab is table of t_fetch_pk_hash_rec;
     
-    l_fetch_tab t_fetch_pk_hash_tab;
-    l_pk_hash_cursor  sys_refcursor;
+    type t_pk_val_tab is table of t_pk_val_rec;
+    l_pk_val_tab t_pk_val_tab;
+    
+    function get_column_xpath(a_join_by_xpath varchar2) return varchar2 is
+      l_column varchar2(32767);
+    begin
+      if a_join_by_xpath is not null then
+        l_column :=  l_ut_owner ||'.ut_compound_data_helper.get_hash(extract(t.item_data,:join_by_xpath).GetClobVal()) pk_hash ';
+      else
+        l_column := ':join_by_xpath pk_hash ';
+      end if;
+      return l_column;
+    end;
+    
   begin
     if not a_other is of (ut_compound_data_value) then
       raise value_error;
@@ -268,60 +285,78 @@ create or replace type body ut_compound_data_value as
     /**
     * Due to incompatibility issues in XML between 11 and 12.2 and 12.1 versions we will prepopulate pk_hash upfront to
     * avoid optimizer incorrectly rewrite and causing NULL error or ORA-600
-    **/
+    **/    
+    -- Pre generate hash minus to leave only onese that are diffrent, for example duplicates or diffrent hash
+    /*if a_join_by_xpath is not null then
+      execute immediate q'[select
+         data_id,item_hash,pk_hash,
+         listagg(extractvalue(tcd.column_value, '/*'), '; ') within GROUP(ORDER BY item_hash) pk_value
+        from
+         (select 
+           ucd.column_value row_data,
+           ]'|| l_ut_owner ||q'[.ut_compound_data_helper.get_hash(extract(ucd.column_value,:join_xpath).GetClobVal()) pk_hash ,
+           t.item_hash,
+           t.data_id
+          from ]' || l_ut_owner || q'[.ut_compound_data_tmp t ,
+          table(xmlsequence(extract(t.item_data,'/*'))) ucd 
+          where data_id = :self_guid or data_id = :other_guid ),
+          table(xmlsequence(extract(row_data ,:join_xpath))) tcd
+          group by pk_hash,item_hash,data_id]' 
+          bulk collect into l_pk_val_tab using a_join_by_xpath,self.data_id, l_other.data_id,a_join_by_xpath;
     
-    if a_join_by_xpath is not null then
-      
-      open l_pk_hash_cursor for q'[select t.data_id,t.item_hash,t.item_data
-                                   from ]'|| l_ut_owner ||q'[.ut_compound_data_tmp t
-                                   ,xmltable('*'
-                                   passing t.item_data
-                                   columns 
-                                   xml_pk_hash clob path ']'|| a_join_by_xpath ||q'['
-                                   ) ucd
-                                  where data_id = :self_guid or data_id = :other_guid ]'                                 
-                                  using self.data_id, l_other.data_id;
-      fetch l_pk_hash_cursor bulk collect into l_fetch_tab;  
-      for pks in 1..l_fetch_tab.COUNT
-      loop
-        execute immediate 'update '|| l_ut_owner ||'.ut_compound_data_tmp 
-        set pk_hash = '|| l_ut_owner ||'.ut_compound_data_helper.get_hash(extract(:item_data,:join_by_xpath).GetClobVal())
-        where item_hash = :item_hash' 
-        using l_fetch_tab(pks).item_data,a_join_by_xpath , l_fetch_tab(pks).item_hash;
-      end loop;
-      
-      close l_pk_hash_cursor;
+         forall pk_vals in 1..l_pk_val_tab.COUNT
+         update ut_compound_data_tmp 
+         set pk_hash = l_pk_val_tab(pk_vals).pk_hash
+             ,pk_value = l_pk_val_tab(pk_vals).pk_value
+         where data_id = l_pk_val_tab(pk_vals).data_id 
+            and item_hash = l_pk_val_tab(pk_vals).item_hash;
     end if;
-    
-    
-    -- Pre generate hash minus to leave only onese that are diffrent, for example duplicates or diffrent hash
-    execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no )
+    */
+
+    execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no)
+                       with calc_pk as
+                       ( select data_id,item_hash, '||get_column_xpath(a_join_by_xpath)||'
+                         from ' || l_ut_owner || '.ut_compound_data_tmp t
+                       )
                        select distinct :diff_id,tmp.item_hash,tmp.pk_hash,tmp.duplicate_no
                        from(
                          (
                            select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no,
-                           pk_hash
-                            from  ' || l_ut_owner || '.ut_compound_data_tmp t
-                           where data_id = :self_guid
+                           pc.pk_hash
+                           from  ' || l_ut_owner || '.ut_compound_data_tmp t,
+                           calc_pk pc
+                           where t.data_id = :self_guid
+                           and   pc.data_id = t.data_id
+                           and   pc.item_hash = t.item_hash
                            minus
                            select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no,
-                           pk_hash
-                            from  ' || l_ut_owner || '.ut_compound_data_tmp t
-                           where data_id = :other_guid
+                           pc.pk_hash
+                           from  ' || l_ut_owner || '.ut_compound_data_tmp t,
+                           calc_pk pc
+                           where t.data_id = :other_guid
+                           and   pc.data_id = t.data_id
+                           and   pc.item_hash = t.item_hash
                          )
                            union all
                          (
                            select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no,
-                           pk_hash
-                            from  ' || l_ut_owner || '.ut_compound_data_tmp t
-                           where data_id = :other_guid
+                           pc.pk_hash
+                            from  ' || l_ut_owner || '.ut_compound_data_tmp t,
+                           calc_pk pc
+                           where t.data_id = :other_guid
+                           and   pc.data_id = t.data_id
+                           and   pc.item_hash = t.item_hash
                            minus
                            select t.item_hash,row_number() over (partition by t.item_hash,t.data_id order by 1,2) duplicate_no,
-                           pk_hash
-                            from  ' || l_ut_owner || '.ut_compound_data_tmp t
-                           where data_id = :self_guid
+                           pc.pk_hash
+                           from  ' || l_ut_owner || '.ut_compound_data_tmp t,
+                           calc_pk pc
+                           where t.data_id = :self_guid
+                           and   pc.data_id = t.data_id
+                           and   pc.item_hash = t.item_hash
                         ))tmp'
-       using l_diff_id, 
+       using a_join_by_xpath,
+             l_diff_id, 
              self.data_id, l_other.data_id,
              l_other.data_id,self.data_id;
     --result is OK only if both are same
diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pkb b/test/core/expectations/compound_data/test_expectations_cursor.pkb
@@ -1113,6 +1113,19 @@ Extra:    <ROW><USERNAME>test</USERNAME><USER_ID>-666</USER_ID></ROW>%]';
     --Assert
     ut.expect(expectations.failed_expectations_data()).to_be_empty();
   end;
-  
+ 
+  procedure cursor_joinby_compare_1000 is
+    l_actual   SYS_REFCURSOR;
+    l_expected SYS_REFCURSOR;
+  begin
+    --Arrange
+    open l_actual for select object_name from all_objects where rownum <=1100;
+    open l_expected for select object_name from all_objects where rownum <=1100;
+    --Act
+    ut3.ut.expect(l_actual).to_equal(l_expected).join_by('OBJECT_NAME');
+    --Assert
+    ut.expect(expectations.failed_expectations_data()).to_be_empty();
+  end;
+ 
 end;
 /
diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pks b/test/core/expectations/compound_data/test_expectations_cursor.pks
@@ -211,5 +211,8 @@ create or replace package test_expectations_cursor is
   --%test( Compare cursors join by composite key)
   procedure cursor_joinby_compare_twocols; 
   
+  --%test( Compare cursors join by single key more than 1000 rows)
+  procedure cursor_joinby_compare_1000;
+  
 end;
 /

Original file line number	Diff line number	Diff line change
`@@ -211,5 +211,8 @@ create or replace package test_expectations_cursor is`
`211`	`211`	`--%test( Compare cursors join by composite key)`
`212`	`212`	`procedure cursor_joinby_compare_twocols;`
`213`	`213`
	`214`	`+ --%test( Compare cursors join by single key more than 1000 rows)`
	`215`	`+ procedure cursor_joinby_compare_1000;`
	`216`	`+`
`214`	`217`	`end;`
`215`	`218`	`/`