Added unique key name as part of compare results

lwasylow · lwasylow · commit 66974e000d77 · 2018-05-11T13:08:00.000+01:00
Small fixes and optimizations
diff --git a/source/expectations/data_values/ut_compound_data_diff_tmp.sql b/source/expectations/data_values/ut_compound_data_diff_tmp.sql
@@ -15,7 +15,6 @@ create global temporary table ut_compound_data_diff_tmp(
   diff_id    raw(128),
   item_no    integer,
   pk_hash    raw(128),
-  pk_value   varchar2(4000),
   item_hash  raw(128),
   duplicate_no integer,
   constraint ut_compound_data_diff_tmp_uk1 unique (diff_id,duplicate_no,item_no,item_hash, pk_hash),
diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb
@@ -186,7 +186,19 @@ create or replace package body ut_compound_data_helper is
 
     return l_results;
   end;
-
+  
+  function get_pk_value (a_join_by_xpath varchar2,a_item_data xmltype) return varchar2 is
+    l_pk_value varchar2(4000);
+  begin
+    select listagg(extractvalue(xmlelement("ROW",column_value),a_join_by_xpath),':') within group ( order by 1)
+    into l_pk_value
+    from table(xmlsequence(extract(a_item_data,'/*/*')));
+    
+    return l_pk_value;
+  exception when no_data_found then
+    return 'null ';
+  end;
+  
   function get_rows_diff(
     a_expected_dataset_guid raw, a_actual_dataset_guid raw, a_diff_id raw,
     a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2,
@@ -195,36 +207,38 @@ create or replace package body ut_compound_data_helper is
     l_column_filter varchar2(32767);
     l_results       tt_row_diffs;
   begin
-    l_column_filter := get_columns_filter(a_exclude_xpath,a_include_xpath);
+    l_column_filter := get_columns_row_filter(a_exclude_xpath,a_include_xpath);
     
    /**
     * Since its unordered search we cannot select max rows from diffs as we miss some comparision records
     * We will restrict output on higher level of select
+    * NO_MERGE hint was introduced to prevent optimizer from merging views and rewriting query which in some cases
+    * lead to second value being null depend on execution plan that been chosen
     **/
-    
     execute immediate q'[
-    with diff_info as (select item_hash,pk_hash,pk_value from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid)
+    with diff_info as (select item_hash,pk_hash,duplicate_no from ut_compound_data_diff_tmp ucdc where diff_id = :diff_guid)
       select rn,diff_type,diffed_row,pk_value from
       (
       select diff_type,diffed_row, dense_rank() over (order by pk_hash) rn,pk_value from
       (
       select diff_type,diffed_row,pk_hash,pk_value from
         (select diff_type,data_item diffed_row,pk_hash,pk_value
          from
-          (select nvl(exp.pk_hash, act.pk_hash) pk_hash,nvl(exp.pk_value, act.pk_value) pk_value,
+          (select /*+NO_MERGE*/ nvl(exp.pk_hash, act.pk_hash) pk_hash,nvl(exp.pk_value, act.pk_value) pk_value,
              xmlserialize(content exp.row_data no indent)  exp_item,
              xmlserialize(content act.row_data no indent)  act_item
              from 
-              (select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no
+              (select ucd.*
                from 
                 (select ucd.column_value row_data,
                  r.item_hash row_hash,
                  r.pk_hash ,
-                 r.pk_value,
+                 r.duplicate_no,
+                 ucd.column_value.getclobval() col_val,
                  ucd.column_value.getRootElement() col_name,
-                 ucd.column_value.getclobval() col_val
+                 ut_compound_data_helper.get_pk_value(:join_xpath,r.item_data) pk_value
                  from 
-                  (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash, i.pk_value
+                  (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash, i.duplicate_no
                    from ut_compound_data_tmp ucd,
                    diff_info i
                    where ucd.data_id = :self_guid
@@ -234,16 +248,17 @@ create or replace package body ut_compound_data_helper is
                 ) ucd
               ) exp
               join (
-                select ucd.*, row_number() over(partition by pk_hash order by row_hash) duplicate_no
+                select ucd.*
                 from 
                  (select ucd.column_value row_data,
                   r.item_hash row_hash,
                   r.pk_hash ,
-                  r.pk_value,
+                  r.duplicate_no,
+                  ucd.column_value.getclobval() col_val,
                   ucd.column_value.getRootElement() col_name,
-                  ucd.column_value.getclobval() col_val
+                  ut_compound_data_helper.get_pk_value(:join_xpath,r.item_data) pk_value
                   from 
-                   (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash, i.pk_value
+                   (select ]'||l_column_filter||q'[, ucd.item_no, ucd.item_hash, i.pk_hash, i.duplicate_no
                     from ut_compound_data_tmp ucd,
                     diff_info i
                     where ucd.data_id = :other_guid
@@ -252,7 +267,8 @@ create or replace package body ut_compound_data_helper is
                    table( xmlsequence( extract(r.item_data,'/*/*') ) ) ucd
                  ) ucd
               )  act
-              on exp.pk_hash = act.pk_hash  and exp.duplicate_no = act.duplicate_no
+              on exp.pk_hash = act.pk_hash  and exp.col_name = act.col_name
+              and exp.duplicate_no = act.duplicate_no
              where dbms_lob.compare(exp.col_val, act.col_val) != 0
               )
             unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:') )
@@ -261,15 +277,17 @@ create or replace package body ut_compound_data_helper is
       select case when exp.pk_hash is null then 'Extra:' else 'Missing:' end as diff_type,
              xmlserialize(content nvl(exp.item_data, act.item_data) no indent) diffed_row,
              coalesce(exp.pk_hash,act.pk_hash) pk_hash,
-             null pk_value
-        from (select extract(ucd.item_data,'/*/*') item_data,i.pk_hash
+             coalesce(exp.pk_value,act.pk_value) pk_value
+        from (select extract(ucd.item_data,'/*/*') item_data,i.pk_hash,
+                ut_compound_data_helper.get_pk_value(:join_by,item_data) pk_value
                 from ut_compound_data_tmp ucd,
                 diff_info i
                where ucd.data_id = :self_guid
                  and ucd.item_hash = i.item_hash
              ) exp
         full outer join (
-              select extract(ucd.item_data,'/*/*') item_data,i.pk_hash
+              select extract(ucd.item_data,'/*/*') item_data,i.pk_hash,
+                ut_compound_data_helper.get_pk_value(:join_by,item_data) pk_value
                 from ut_compound_data_tmp ucd,
                 diff_info i
                where ucd.data_id = :other_guid
@@ -282,10 +300,13 @@ create or replace package body ut_compound_data_helper is
       ]'
     bulk collect into l_results
     using a_diff_id,
+    a_join_by_xpath,
     a_exclude_xpath, a_include_xpath, a_expected_dataset_guid,
+    a_join_by_xpath,
     a_exclude_xpath, a_include_xpath, a_actual_dataset_guid,
-    a_expected_dataset_guid, a_actual_dataset_guid,
+    a_join_by_xpath,a_expected_dataset_guid,a_join_by_xpath, a_actual_dataset_guid,
     a_max_rows;
+        
     return l_results;
   end;
 
diff --git a/source/expectations/data_values/ut_compound_data_helper.pks b/source/expectations/data_values/ut_compound_data_helper.pks
@@ -60,6 +60,8 @@ create or replace package ut_compound_data_helper authid definer is
     a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2
   ) return tt_row_diffs;
 
+  function get_pk_value (a_join_by_xpath varchar2,a_item_data xmltype) return varchar2;
+  
   function get_rows_diff(
     a_expected_dataset_guid raw, a_actual_dataset_guid raw, a_diff_id raw,
     a_max_rows integer, a_exclude_xpath varchar2, a_include_xpath varchar2,
diff --git a/source/expectations/data_values/ut_compound_data_value.tpb b/source/expectations/data_values/ut_compound_data_value.tpb
@@ -251,21 +251,6 @@ create or replace type body ut_compound_data_value as
     l_row_diffs       ut_compound_data_helper.tt_row_diffs;
     c_max_rows        constant integer := 20;
     
-    function get_pk_value(a_join_by_xpath varchar2) return varchar2 is
-      l_column varchar2(32767);
-    begin
-      /* due to possibility of key being to columns we cannot use xmlextractvalue
-         usage of xmlagg is possible however it greatly complicates code and performance is impacted.
-         xpath to be looked at or regex
-      */
-      if a_join_by_xpath is not null then
-        l_column :=  ' extract(t.item_data,:join_by_xpath).GetStringVal() pk_value ';
-      else
-        l_column := ' :join_by_xpath pk_value';
-      end if;
-      return l_column;
-    end;
-
     function get_column_pk_hash(a_join_by_xpath varchar2) return varchar2 is
       l_column varchar2(32767);
     begin
@@ -313,35 +298,35 @@ create or replace type body ut_compound_data_value as
                        using a_exclude_xpath, a_include_xpath,a_join_by_xpath,self.data_id, l_other.data_id;
     
     /* Peform minus on two sets two get diffrences that will be used later on to print results */
-    execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no,pk_value)
+    execute immediate 'insert into ' || l_ut_owner || '.ut_compound_data_diff_tmp ( diff_id,item_hash,pk_hash,duplicate_no)
                        with source_data as
                        ( select t.data_id,t.item_hash,row_number() over (partition by t.pk_hash,t.item_hash,t.data_id order by 1,2) duplicate_no,
-                           pk_hash, '||get_pk_value(a_join_by_xpath)||'
+                           pk_hash
                            from  ' || l_ut_owner || '.ut_compound_data_tmp t
                            where data_id = :self_guid or data_id = :other_guid
                         )           
-                       select distinct :diff_id,tmp.item_hash,tmp.pk_hash,tmp.duplicate_no,pk_value
+                       select distinct :diff_id,tmp.item_hash,tmp.pk_hash,tmp.duplicate_no
                        from(
                          (
-                           select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value
+                           select t.item_hash,t. duplicate_no,t.pk_hash
                            from  source_data t
                            where t.data_id = :self_guid
                            minus
-                           select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value
+                           select t.item_hash,t. duplicate_no,t.pk_hash
                            from  source_data t
                            where t.data_id = :other_guid
                          )
                            union all
                          (
-                           select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value
+                           select t.item_hash,t. duplicate_no,t.pk_hash
                            from  source_data t
                            where t.data_id = :other_guid
                            minus
-                           select t.item_hash,t. duplicate_no,t.pk_hash, t.pk_value
+                           select t.item_hash,t. duplicate_no,t.pk_hash
                            from  source_data t
                            where t.data_id = :self_guid
                         ))tmp'
-       using a_join_by_xpath,self.data_id, l_other.data_id,
+       using self.data_id, l_other.data_id,
              l_diff_id, 
              self.data_id, l_other.data_id,
              l_other.data_id,self.data_id;
diff --git a/source/expectations/data_values/ut_data_value_refcursor.tpb b/source/expectations/data_values/ut_data_value_refcursor.tpb
@@ -198,7 +198,7 @@ create or replace type body ut_data_value_refcursor as
         ut_utils.append_to_clob(l_result,chr(10) || 'Unable to join sets:' || chr(10));
         for i in 1 .. l_missing_pk.count loop
           l_results.extend;
-          ut_utils.append_to_clob(l_result, get_missing_key_message(l_missing_pk(i)));
+          ut_utils.append_to_clob(l_result, get_missing_key_message(l_missing_pk(i))|| chr(10));
         end loop;    
       end if;
     else
diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pkb b/test/core/expectations/compound_data/test_expectations_cursor.pkb
diff --git a/test/core/expectations/compound_data/test_expectations_cursor.pks b/test/core/expectations/compound_data/test_expectations_cursor.pks
