Change behaviour to compare columns regardless of column position in …

…ref cursor.
utPLSQL · jgebal · Mar 15, 2019 · Oct 6, 2018 · Oct 6, 2018 · Oct 6, 2018
commit d8525d60c8bb94ef1a1ad2a11d274078a5035e3f
diff --git a/docs/userguide/expectations.md b/docs/userguide/expectations.md
@@ -680,13 +680,36 @@ utPLSQL is capable of comparing compound data-types including:
 
 ### Notes on comparison of compound data
 - Compound data can contain elements of any data-type. This includes blob, clob, object type, nested table, varray or even a nested-cursor within a cursor.   
-- Cursors, nested table and varray types are compared as **ordered lists of elements**. If order of elements differ, expectation will fail.   
+
+- Nested table and varray types are compared as **ordered lists of elements**. If order of elements differ, expectation will fail.   
+
+- Cursors are compared as **unordered list of elements** by default. If order of elements is of importance the option has to be passed to enforce column order comparison  `ordered_columns` e.g.
+
+    ```sql
+    procedure ut_refcursors1 is
+        l_actual   sys_refcursor;
+        l_expected sys_refcursor;
+        l_expected_message varchar2(32767);
+        l_actual_message   varchar2(32767);
+    begin
+        open l_actual for select 1 user_id,'s' a_col,'test' username from dual;
+        open l_expected for select 'test' username,'s' a_col,1 user_id from dual;
+        --Act
+          ut3.ut.expect(l_actual).to_equal(l_expected).join_by('USER_ID').ordered_columns;
+    end;
+    ```
+
 - Comparison of compound data is data-type aware. So a column `ID NUMBER` in a cursor is not the same as `ID VARCHAR2(100)`, even if they both hold the same numeric values.
+
 - Comparison of cursor columns containing `DATE` will only compare date part **and ignore time** by default. See [Comparing cursor data containing DATE fields](#comparing-cursor-data-containing-date-fields) to check how to enable date-time comparison in cursors.
+
 - Comparison of cursor returning `TIMESTAMP` **columns** against cursor returning `TIMESTAMP` **bind variables** requires variables to be casted to proper precision. This is an Oracle SQL - PLSQL compatibility issue and usage of CAST is the only known workaround for now.
-See [Comparing cursor data containing TIMESTAMP bind variables](#comparing-cursor-data-containing-timestamp-bind-variables) for examples.    
+  See [Comparing cursor data containing TIMESTAMP bind variables](#comparing-cursor-data-containing-timestamp-bind-variables) for examples.    
+
 - To compare nested table/varray type you need to convert it to `anydata` by using `anydata.convertCollection()`  
+
 - To compare object type you need to convert it to `anydata` by using `anydata.convertObject()`  
+
 - It is possible to compare PL/SQL records, collections, varrays and associative arrays. To compare this types of data, use cursor comparison feature of utPLSQL and TABLE operator in SQL query
     - On Oracle 11g Release 2 - pipelined table functions are needed (see section [Implicit (Shadow) Types in this artcile](https://oracle-base.com/articles/misc/pipelined-table-functions))
     - On Oracle 12c and above - use [TABLE function on nested tables/varrays/associative arrays of PL/SQL records](https://oracle-base.com/articles/12c/using-the-table-operator-with-locally-defined-types-in-plsql-12cr1) 
@@ -768,7 +791,7 @@ create or replace package body test_cursor_compare as
         from dual union all
       select 'M' AS GENDER, 'LUKE' as FIRST_NAME, 'SKYWALKER' AS LAST_NAME, 2 as ID, '1000' AS SALARY
         from dual;
-    ut.expect(l_actual).to_equal(l_expected);
+    ut.expect(l_actual).to_equal(l_expected).ordered_columns;
   end;
 end;
 /

diff --git a/source/expectations/data_values/ut_compound_data_helper.pkb b/source/expectations/data_values/ut_compound_data_helper.pkb
@@ -46,8 +46,8 @@ create or replace package body ut_compound_data_helper is
     end if;
     return l_filter;
   end;
-
-  function get_columns_diff(a_expected ut_cursor_column_tab, a_actual ut_cursor_column_tab) 
+    
+  function get_columns_diff_ordered(a_expected ut_cursor_column_tab, a_actual ut_cursor_column_tab) 
   return tt_column_diffs is
     l_column_filter  varchar2(32767);
     l_sql            varchar2(32767);
@@ -88,6 +88,53 @@ create or replace package body ut_compound_data_helper is
     return l_results;
   end;
 
+  function get_columns_diff_unordered(a_expected ut_cursor_column_tab, a_actual ut_cursor_column_tab) 
+  return tt_column_diffs is
+    l_column_filter  varchar2(32767);
+    l_sql            varchar2(32767);
+    l_results        tt_column_diffs;
+  begin
+    with 
+      expected_cols as 
+      (select access_path exp_column_name,column_position exp_col_pos,
+      replace(column_type,'VARCHAR2','CHAR') exp_col_type_compare, column_type exp_col_type
+      from table(a_expected)),
+      actual_cols as
+      (select access_path act_column_name,column_position act_col_pos,
+      replace(column_type,'VARCHAR2','CHAR') act_col_type_compare, column_type act_col_type
+      from table(a_actual)),
+      joined_cols as
+      (select e.*,a.*
+      from expected_cols e
+      full outer join actual_cols a on e.exp_column_name = a.act_column_name)
+      select case
+               when exp_col_pos is null and act_col_pos is not null then '+'
+               when exp_col_pos is not null and act_col_pos is null then '-'
+               when exp_col_type_compare != act_col_type_compare then 't'
+               else 'p'
+             end as diff_type,
+             exp_column_name, exp_col_type, exp_col_pos,
+             act_column_name, act_col_type, act_col_pos
+        bulk collect into l_results
+        from joined_cols
+             --column is unexpected (extra) or missing
+       where act_col_pos is null or exp_col_pos is null
+          --column type is not matching (except CHAR/VARCHAR2)
+          or act_col_type_compare != exp_col_type_compare
+       order by exp_col_pos, act_col_pos;
+    return l_results;
+  end;
+
+  function get_columns_diff(a_expected ut_cursor_column_tab, a_actual ut_cursor_column_tab,a_order_enforced boolean := false) 
+  return tt_column_diffs is
+  begin
+    if a_order_enforced then
+      return get_columns_diff_ordered(a_expected,a_actual);
+    else 
+      return get_columns_diff_unordered(a_expected,a_actual);
+    end if;
+  end;
+
   function get_pk_value (a_join_by_xpath varchar2,a_item_data xmltype) return clob is
     l_pk_value clob;
   begin
@@ -218,10 +265,10 @@ create or replace package body ut_compound_data_helper is
     ut_utils.append_to_clob(a_sql_stmt, l_sql_stmt);
   end;
 
-  procedure gen_sql_pieces_out_of_cursor(a_data_info ut_data_value_refcursor,a_pk_table ut_varchar2_list, a_xml_stmt out nocopy clob, 
+  procedure gen_sql_pieces_out_of_cursor(a_data_info ut_cursor_column_tab,a_pk_table ut_varchar2_list, a_xml_stmt out nocopy clob, 
   a_select_stmt out nocopy clob  ,a_partition_stmt out nocopy clob, a_equal_stmt out nocopy clob, a_join_by_stmt out nocopy clob,
   a_not_equal_stmt out nocopy clob) is
-    l_cursor_info ut_cursor_column_tab := a_data_info.cursor_details.cursor_info;
+    l_cursor_info ut_cursor_column_tab := a_data_info;
     l_partition_tmp clob;
     l_col_name varchar2(100);
   begin
@@ -305,7 +352,7 @@ create or replace package body ut_compound_data_helper is
 
   begin
     dbms_lob.createtemporary(l_compare_sql, true);
-    gen_sql_pieces_out_of_cursor(a_other, a_join_by_list, 
+    gen_sql_pieces_out_of_cursor(a_other.cursor_details.cursor_info, a_join_by_list, 
       l_xmltable_stmt, l_select_stmt, l_partition_stmt, l_equal_stmt, 
       l_join_on_stmt, l_not_equal_stmt);
 
@@ -374,9 +421,8 @@ create or replace package body ut_compound_data_helper is
 
   function get_rows_diff_by_sql(a_act_cursor_info ut_cursor_column_tab,a_exp_cursor_info ut_cursor_column_tab, 
     a_expected_dataset_guid raw, a_actual_dataset_guid raw, a_diff_id raw,
-    a_join_by_list ut_varchar2_list, a_unordered boolean
+    a_join_by_list ut_varchar2_list, a_unordered boolean, a_enforce_column_order boolean := false
   ) return tt_row_diffs is
-
     l_act_col_filter varchar2(32767);
     l_exp_col_filter varchar2(32767);
     l_act_extract_xpath  varchar2(32767):= ut_utils.to_xpath(get_column_extract_path(a_act_cursor_info));
@@ -412,25 +458,28 @@ create or replace package body ut_compound_data_helper is
       select rn, diff_type, diffed_row, pk_value
       ,case when diff_type = 'Actual:' then 1 else 2 end rnk
       ,1 final_order
+      ,col_name
       from ( ]';
 
     if a_unordered then 
-      l_sql := l_sql || q'[select rn, diff_type, xmlserialize(content data_item no indent) diffed_row, pk_value pk_value
+      l_sql := l_sql || q'[select rn, diff_type, xmlserialize(content data_item no indent) diffed_row, pk_value,col_name
       from 
-        (select nvl(exp.rn, act.rn) rn, nvl(exp.pk_value, act.pk_value) pk_value, exp.col  exp_item, act.col  act_item       
+        (select nvl(exp.rn, act.rn) rn, nvl(exp.pk_value, act.pk_value) pk_value, exp.col  exp_item, act.col  act_item ,
+        nvl(exp.col_name,act.col_name) col_name
         from exp join act on exp.rn = act.rn and exp.col_name = act.col_name
         where dbms_lob.compare(exp.col_val, act.col_val) != 0)
         unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:') 
       ))]';
     else
-    l_sql := l_sql || q'[ select rn, diff_type, xmlserialize(content data_item no indent) diffed_row, null pk_value
+    l_sql := l_sql || q'[ select rn, diff_type, xmlserialize(content data_item no indent) diffed_row, null pk_value,col_name
       from 
         (select nvl(exp.rn, act.rn) rn,
           xmlagg(exp.col order by exp.col_no) exp_item,
-          xmlagg(act.col order by act.col_no) act_item
+          xmlagg(act.col order by act.col_no) act_item,
+          max(nvl(exp.col_name,act.col_name)) col_name
         from exp exp join act act on exp.rn = act.rn and exp.col_name = act.col_name
         where dbms_lob.compare(exp.col_val, act.col_val) != 0
-        group by exp.rn, act.rn
+        group by (exp.rn, act.rn)
         )
         unpivot ( data_item for diff_type in (exp_item as 'Expected:', act_item as 'Actual:'))
       )]';
@@ -443,14 +492,26 @@ create or replace package body ut_compound_data_helper is
       nvl2(:join_by,ut_compound_data_helper.get_pk_value(:join_by,case when exp_data_id is null then act_item_data else exp_item_data end),null) pk_value
       ,case when exp_data_id is null then 1 else 2 end rnk
       ,2 final_order
+      ,null col_name
     from   ut_compound_data_diff_tmp i
     where  diff_id = :diff_id 
     and    act_data_id is null or exp_data_id is null
    )
-   order by final_order, 
-   case when final_order = 1 then rn else rnk end,
-   case when final_order = 1 then rnk else rn end ]';
+   order by final_order,]';
 
+   if a_enforce_column_order then
+     l_sql := l_sql ||q'[case when final_order = 1 then rn else rnk end,
+     case when final_order = 1 then rnk else rn end ]';
+   elsif not(a_enforce_column_order) and not(a_unordered) then
+     l_sql := l_sql ||q'[case when final_order = 1 then rn else rnk end,
+     case when final_order = 1 then rnk else rn end ]';    
+   elsif a_unordered then
+     l_sql := l_sql ||q'[case when final_order = 1 then col_name else to_char(rnk) end,
+     case when final_order = 1 then to_char(rn) else col_name end,
+     case when final_order = 1 then to_char(rnk) else col_name end
+     ]';  
+   end if;
+
    execute immediate l_sql
    bulk collect into l_results
     using l_exp_extract_xpath,l_join_xpath,a_diff_id, a_expected_dataset_guid,

diff --git a/source/expectations/data_values/ut_compound_data_helper.pks b/source/expectations/data_values/ut_compound_data_helper.pks
@@ -57,13 +57,13 @@ create or replace package ut_compound_data_helper authid definer is
     );
 
   type t_diff_tab is table of t_diff_rec;
-
+          
   function get_columns_filter(
     a_exclude_xpath varchar2, a_include_xpath varchar2,
     a_table_alias varchar2 := 'ucd', a_column_alias varchar2 := 'item_data'
   ) return varchar2;
 
-  function get_columns_diff(a_expected ut_cursor_column_tab, a_actual ut_cursor_column_tab) 
+  function get_columns_diff(a_expected ut_cursor_column_tab, a_actual ut_cursor_column_tab,a_order_enforced boolean := false) 
   return tt_column_diffs;
 
  function get_pk_value (a_join_by_xpath varchar2,a_item_data xmltype) return clob;
@@ -75,7 +75,7 @@ create or replace package ut_compound_data_helper authid definer is
 
   function get_rows_diff_by_sql(a_act_cursor_info ut_cursor_column_tab,a_exp_cursor_info ut_cursor_column_tab, 
     a_expected_dataset_guid raw, a_actual_dataset_guid raw, a_diff_id raw,
-    a_join_by_list ut_varchar2_list, a_unordered boolean
+    a_join_by_list ut_varchar2_list, a_unordered boolean, a_enforce_column_order boolean := false
   ) return tt_row_diffs;
 
   subtype t_hash  is raw(128);

diff --git a/source/expectations/data_values/ut_compound_data_value.tpb b/source/expectations/data_values/ut_compound_data_value.tpb
@@ -223,7 +223,7 @@ create or replace type body ut_compound_data_value as
    end loop;
 
    ut_compound_data_helper.set_rows_diff(l_sql_rowcount); 
-        --result is OK only if both are same  
+   --result is OK only if both are same  
    if l_sql_rowcount = 0 and ( self.elements_count = l_other.elements_count or a_inclusion_compare )then
      l_result := 0; 
    else

diff --git a/source/expectations/data_values/ut_cursor_details.tpb b/source/expectations/data_values/ut_cursor_details.tpb
@@ -2,13 +2,21 @@ create or replace type body ut_cursor_details as
 
   order member function compare(a_other ut_cursor_details) return integer is
    l_diffs integer;
-  begin     
-    select count(1) into l_diffs
-    from table(self.cursor_info) a full outer join table(a_other.cursor_info) e
-    on  ( decode(a.parent_name,e.parent_name,1,0)= 1 and a.column_name = e.column_name and 
-          REPLACE(a.column_type,'VARCHAR2','CHAR') =  REPLACE(e.column_type,'VARCHAR2','CHAR')
-         and  a.column_position = e.column_position )
-    where a.column_name is null or e.column_name is null;   
+  begin   
+    if self.is_column_order_enforced = 1 then
+      select count(1) into l_diffs
+      from table(self.cursor_info) a full outer join table(a_other.cursor_info) e
+      on  ( decode(a.parent_name,e.parent_name,1,0)= 1 and a.column_name = e.column_name and 
+        REPLACE(a.column_type,'VARCHAR2','CHAR') =  REPLACE(e.column_type,'VARCHAR2','CHAR')
+       and  a.column_position = e.column_position )
+      where a.column_name is null or e.column_name is null;  
+    else
+      select count(1) into l_diffs
+      from table(self.cursor_info) a full outer join table(a_other.cursor_info) e
+      on  ( decode(a.parent_name,e.parent_name,1,0)= 1 and a.column_name = e.column_name and 
+        REPLACE(a.column_type,'VARCHAR2','CHAR') =  REPLACE(e.column_type,'VARCHAR2','CHAR'))
+      where a.column_name is null or e.column_name is null;   
+    end if;
     return l_diffs;
   end;
 
@@ -221,5 +229,10 @@ create or replace type body ut_cursor_details as
       return;
    end;
 
+  member procedure ordered_columns(self in out nocopy ut_cursor_details,a_ordered_columns boolean := false) is
+  begin
+    self.is_column_order_enforced := ut_utils.boolean_to_int(a_ordered_columns);
+  end;
+
 end;
 /
diff --git a/source/expectations/data_values/ut_cursor_details.tps b/source/expectations/data_values/ut_cursor_details.tps
@@ -1,6 +1,7 @@
 create or replace type ut_cursor_details force authid current_user as object
 (
    cursor_info ut_cursor_column_tab,
+   is_column_order_enforced number(1,0),
    order member function compare(a_other ut_cursor_details) return integer,
    member procedure get_anytype_members_info(a_anytype anytype, a_attribute_typecode out pls_integer,
     a_schema_name out varchar2, a_type_name out varchar2, a_len out pls_integer,a_elements_count out pls_integer),
@@ -15,6 +16,7 @@ create or replace type ut_cursor_details force authid current_user as object
    member function get_user_defined_type(a_data anydata) return anytype,
    constructor function ut_cursor_details(self in out nocopy ut_cursor_details) return self as result,
    constructor function ut_cursor_details(self in out nocopy ut_cursor_details,a_cursor_number in number)
-      return self as result
+      return self as result,
+   member procedure ordered_columns(self in out nocopy ut_cursor_details,a_ordered_columns boolean := false)
 )
 /
diff --git a/source/expectations/data_values/ut_data_value_refcursor.tpb b/source/expectations/data_values/ut_data_value_refcursor.tpb
@@ -141,6 +141,8 @@ create or replace type body ut_data_value_refcursor as
     l_row_diffs         ut_compound_data_helper.tt_row_diffs;
     l_message           varchar2(32767);
 
+    l_column_order_enforce boolean := ut_utils.int_to_boolean(self.cursor_details.is_column_order_enforced);
+
     function get_col_diff_text(a_col ut_compound_data_helper.t_column_diffs) return varchar2 is
     begin
       return
@@ -208,8 +210,8 @@ create or replace type body ut_data_value_refcursor as
     dbms_lob.createtemporary(l_result,true);
     --diff columns
     if not self.is_null and not l_actual.is_null then
-      l_column_diffs := ut_compound_data_helper.get_columns_diff(self.cursor_details.cursor_info,l_actual.cursor_details.cursor_info);
-
+      l_column_diffs := ut_compound_data_helper.get_columns_diff(self.cursor_details.cursor_info,l_actual.cursor_details.cursor_info,l_column_order_enforce);
+    
       if l_column_diffs.count > 0 then
         ut_utils.append_to_clob(l_result,chr(10) || 'Columns:' || chr(10));
       end if;
@@ -234,9 +236,9 @@ create or replace type body ut_data_value_refcursor as
     -- First tell how many rows are different
     l_diff_row_count := ut_compound_data_helper.get_rows_diff_count; 
     l_results := ut_utils.t_clob_tab();
-    if l_diff_row_count > 0  then
+      if l_diff_row_count > 0  then
         l_row_diffs := ut_compound_data_helper.get_rows_diff_by_sql(
-              l_exp_cols,l_act_cols, self.data_id, l_actual.data_id, l_diff_id,a_join_by_list , a_unordered);
+              l_exp_cols,l_act_cols, self.data_id, l_actual.data_id, l_diff_id,a_join_by_list , a_unordered, l_column_order_enforce);
         l_message := chr(10)
                      ||'Rows: [ ' || l_diff_row_count ||' differences'
                      ||  case when  l_diff_row_count > c_max_rows and l_row_diffs.count > 0 then ', showing first '||c_max_rows end
@@ -248,7 +250,7 @@ create or replace type body ut_data_value_refcursor as
         end loop;
         ut_utils.append_to_clob(l_result,l_results);
       else
-        l_message:= chr(10)||'Rows: [  all different ]'||chr(10)||'  All rows are different as the columns are not matching.';
+        l_message:= chr(10)||'Rows: [  all different ]'||chr(10)||'  All rows are different as the columns position is not matching.';
         ut_utils.append_to_clob( l_result, l_message );
       end if;   
     else
@@ -304,11 +306,12 @@ create or replace type body ut_data_value_refcursor as
     return self.elements_count = 0;
   end;
 
-  member function filter_cursor (a_exclude_xpath ut_varchar2_list, a_include_xpath ut_varchar2_list) return ut_data_value_refcursor is
+  member function update_cursor_details (a_exclude_xpath ut_varchar2_list, a_include_xpath ut_varchar2_list,a_ordered_columns boolean := false) return ut_data_value_refcursor is
     l_result ut_data_value_refcursor := self;
   begin   
     if l_result.cursor_details.cursor_info is not null then
       l_result.cursor_details.cursor_info := ut_compound_data_helper.inc_exc_columns_from_cursor(l_result.cursor_details.cursor_info,a_exclude_xpath,a_include_xpath);
+      l_result.cursor_details.ordered_columns(a_ordered_columns);
     end if;    
     return l_result;
   end;

diff --git a/source/expectations/data_values/ut_data_value_refcursor.tps b/source/expectations/data_values/ut_data_value_refcursor.tps
@@ -38,5 +38,6 @@ create or replace type ut_data_value_refcursor under ut_compound_data_value(
   overriding member function compare_implementation(a_other ut_data_value, a_unordered boolean, a_inclusion_compare boolean := false, a_is_negated boolean := false, 
                                          a_join_by_list ut_varchar2_list:=ut_varchar2_list()) return integer,
   overriding member function is_empty return boolean,
-  member function filter_cursor (a_exclude_xpath ut_varchar2_list, a_include_xpath ut_varchar2_list) return ut_data_value_refcursor)
+  member function update_cursor_details (a_exclude_xpath ut_varchar2_list, a_include_xpath ut_varchar2_list,a_ordered_columns boolean := false) return ut_data_value_refcursor
+)
 /