Merge pull request #417 from jgebal/feature/table_access_iprovements

jgebal · web-flow · commit 6a527c6e305e · 2017-07-28T01:55:43.000+01:00
Improved performance and stability of access to internal framework tables
diff --git a/source/api/ut_runner.pkb b/source/api/ut_runner.pkb
@@ -43,13 +43,6 @@ create or replace package body ut_runner is
     return ut_utils.gc_version;
   end;
 
-  procedure cleanup_temp_tables is
-    pragma autonomous_transaction;
-  begin
-    delete from ut_cursor_data where 1 = 1;
-    commit;
-  end;
-
   procedure run(
     a_paths ut_varchar2_list, a_reporters ut_reporters, a_color_console boolean := false,
     a_coverage_schemes ut_varchar2_list := null, a_source_file_mappings ut_file_mappings := null, a_test_file_mappings ut_file_mappings := null,
@@ -78,11 +71,11 @@ create or replace package body ut_runner is
       );
       l_items_to_run.do_execute(l_listener);
 
-      cleanup_temp_tables;
+      ut_utils.cleanup_temp_tables;
       ut_output_buffer.close(l_listener.reporters);
     exception
       when others then
-        cleanup_temp_tables;
+        ut_utils.cleanup_temp_tables;
         ut_output_buffer.close(l_listener.reporters);
         dbms_output.put_line(dbms_utility.format_error_backtrace);
         dbms_output.put_line(dbms_utility.format_error_stack);
diff --git a/source/core/coverage/ut_coverage.pkb b/source/core/coverage/ut_coverage.pkb
@@ -28,7 +28,7 @@ create or replace package body ut_coverage is
   -- The subquery is optimized by:
   -- - COALESCE function -> it will execute only for TRIGGERS
   -- - scalar subquery cache -> it will only execute once for one trigger source code.
-  function populate_sources_tmp_table(a_coverage_options ut_coverage_options) return varchar2 is
+  function get_populate_sources_tmp_sql(a_coverage_options ut_coverage_options) return varchar2 is
     l_result varchar2(32767);
     l_full_name varchar2(100);
   begin
@@ -98,6 +98,31 @@ create or replace package body ut_coverage is
     return l_result;
   end;
 
+  function is_tmp_table_populated return boolean is
+    l_result integer;
+  begin
+    select 1 into l_result from ut_coverage_sources_tmp where rownum = 1;
+    return (l_result = 1);
+  exception
+    when no_data_found then
+      return false;
+  end;
+
+  procedure populate_tmp_table(a_coverage_options ut_coverage_options, a_skipped_objects ut_object_names) is
+    pragma autonomous_transaction;
+    l_schema_names     ut_varchar2_rows;
+  begin
+    delete from ut_coverage_sources_tmp;
+    l_schema_names := coalesce(a_coverage_options.schema_names, ut_varchar2_rows(sys_context('USERENV','CURRENT_SCHEMA')));
+    if a_coverage_options.file_mappings is not empty then
+      execute immediate get_populate_sources_tmp_sql(a_coverage_options) using a_coverage_options.file_mappings, a_skipped_objects, a_coverage_options.include_objects;
+    else
+      execute immediate get_populate_sources_tmp_sql(a_coverage_options) using l_schema_names, a_skipped_objects, a_coverage_options.include_objects;
+    end if;
+    commit;
+  end;
+
+
   /**
   * Public functions
   */
@@ -132,41 +157,24 @@ create or replace package body ut_coverage is
   end;
 
   function get_coverage_data(a_coverage_options ut_coverage_options) return t_coverage is
-
-    pragma autonomous_transaction;
-
-    type t_coverage_row is record(
-      name          varchar2(500),
-      line_number   integer,
-      total_occur   number(38,0)
-    );
-    type tt_coverage_rows is table of t_coverage_row;
     l_line_calls       ut_coverage_helper.unit_line_calls;
     l_result           t_coverage;
     l_new_unit         t_unit_coverage;
     l_skipped_objects  ut_object_names := ut_object_names();
 
     type t_source_lines is table of binary_integer;
-    l_source_lines     t_source_lines;
     line_no            binary_integer;
-    l_schema_names     ut_varchar2_rows;
-    l_query            varchar2(32767);
   begin
-    l_schema_names := coalesce(a_coverage_options.schema_names, ut_varchar2_rows(sys_context('USERENV','CURRENT_SCHEMA')));
 
     if not ut_coverage_helper.is_develop_mode() then
       --skip all the utplsql framework objects and all the unit test packages that could potentially be reported by coverage.
       l_skipped_objects := ut_utils.get_utplsql_objects_list() multiset union all coalesce(a_coverage_options.exclude_objects, ut_object_names());
     end if;
 
     --prepare global temp table with sources
-    delete from ut_coverage_sources_tmp;
-    if a_coverage_options.file_mappings is not null and a_coverage_options.file_mappings.count > 0 then
-      execute immediate populate_sources_tmp_table(a_coverage_options) using a_coverage_options.file_mappings, l_skipped_objects, a_coverage_options.include_objects;
-    else
-      execute immediate populate_sources_tmp_table(a_coverage_options) using l_schema_names, l_skipped_objects, a_coverage_options.include_objects;
+    if not is_tmp_table_populated() or ut_coverage_helper.is_develop_mode() then
+      populate_tmp_table(a_coverage_options, l_skipped_objects);
     end if;
-    commit;
 
     for src_object in (
       select o.owner, o.name, o.full_name, max(o.line) lines_count,
@@ -224,7 +232,6 @@ create or replace package body ut_coverage is
 
     end loop;
 
-    commit;
     return l_result;
   end get_coverage_data;
 
diff --git a/source/core/coverage/ut_coverage_helper.pkb b/source/core/coverage/ut_coverage_helper.pkb
@@ -16,32 +16,36 @@ create or replace package body ut_coverage_helper is
   limitations under the License.
   */
 
-  g_coverage_id integer;
-  g_develop_mode boolean;
+  g_coverage_id  integer;
+  g_develop_mode boolean not null := false;
+  g_is_started   boolean not null := false;
 
-  function  is_develop_mode return boolean is
+  function is_develop_mode return boolean is
   begin
     return g_develop_mode;
   end;
 
   procedure coverage_start_internal(a_run_comment varchar2)  is
   begin
     dbms_profiler.start_profiler(run_comment => a_run_comment, run_number => g_coverage_id);
+    g_is_started := true;
     coverage_pause();
   end;
 
   procedure coverage_start(a_run_comment varchar2) is
   begin
-    if g_develop_mode is null then
+    if not g_is_started then
       g_develop_mode := false;
       coverage_start_internal(a_run_comment);
     end if;
   end;
 
   procedure coverage_start_develop is
   begin
-    g_develop_mode := true;
-    coverage_start_internal('utPLSQL Code coverage run in development MODE '||ut_utils.to_string(systimestamp));
+    if not g_is_started then
+      g_develop_mode := true;
+      coverage_start_internal('utPLSQL Code coverage run in development MODE '||ut_utils.to_string(systimestamp));
+    end if;
   end;
 
   procedure coverage_pause is
@@ -59,17 +63,18 @@ create or replace package body ut_coverage_helper is
   end;
 
   procedure coverage_stop is
-    l_return_code binary_integer;
   begin
     if not g_develop_mode then
-      l_return_code := dbms_profiler.stop_profiler();
+      g_is_started := false;
+      dbms_profiler.stop_profiler();
     end if;
   end;
 
   procedure coverage_stop_develop is
-    l_return_code binary_integer;
   begin
-    l_return_code := dbms_profiler.stop_profiler();
+    g_develop_mode := false;
+    g_is_started := false;
+    dbms_profiler.stop_profiler();
   end;
 
   function get_raw_coverage_data(a_object_owner varchar2, a_object_name varchar2) return unit_line_calls is
@@ -81,7 +86,13 @@ create or replace package body ut_coverage_helper is
     l_tmp_data coverage_rows;
     l_results  unit_line_calls;
   begin
-      select d.line#, d.total_occur
+      select d.line#,
+        -- This transformation addresses two issues:
+        -- 1. dbms_profiler shows multiple unit_number for single code unit;
+        --    to address this, we take a sum od all units by name
+        -- 2. some lines show 0 total_occur while they were executed (time > 0)
+        --    in this case we show 1 to indicate that there was execution even if we don't know how many there were
+        case when sum(d.total_occur) = 0 and sum(d.total_time) > 0 then 1 else sum(d.total_occur) end total_occur
       bulk collect into l_tmp_data
         from plsql_profiler_units u
         join plsql_profiler_data d
@@ -91,7 +102,8 @@ create or replace package body ut_coverage_helper is
          and u.unit_owner = a_object_owner
          and u.unit_name = a_object_name
          --exclude specification
-         and u.unit_type not in ('PACKAGE SPEC', 'TYPE SPEC', 'ANONYMOUS BLOCK');
+         and u.unit_type not in ('PACKAGE SPEC', 'TYPE SPEC', 'ANONYMOUS BLOCK')
+       group by d.line#;
     for i in 1 .. l_tmp_data.count loop
       l_results(l_tmp_data(i).line) := l_tmp_data(i).calls;
     end loop;
diff --git a/source/core/coverage/ut_coverage_helper.pks b/source/core/coverage/ut_coverage_helper.pks
@@ -17,7 +17,7 @@ create or replace package ut_coverage_helper authid definer is
   */
 
   --table of line calls indexed by line number
-  -- table is sparse!!!
+  --!!! this table is sparse!!!
   type unit_line_calls is table of number(38,0) index by binary_integer;
 
   function  is_develop_mode return boolean;
diff --git a/source/core/ut_utils.pkb b/source/core/ut_utils.pkb
@@ -367,5 +367,13 @@ create or replace package body ut_utils is
     return l_xpath;
   end;
 
+  procedure cleanup_temp_tables is
+    pragma autonomous_transaction;
+  begin
+    execute immediate 'truncate table ut_cursor_data';
+    execute immediate 'truncate table ut_coverage_sources_tmp$';
+    commit;
+  end;
+
 end ut_utils;
 /
diff --git a/source/core/ut_utils.pks b/source/core/ut_utils.pks
@@ -223,5 +223,7 @@ create or replace package ut_utils authid definer is
 
   function to_xpath(a_list ut_varchar2_list, a_ancestors varchar2 := '/*/') return varchar2;
 
+  procedure cleanup_temp_tables;
+
 end ut_utils;
 /
diff --git a/tests/RunAll.cmd b/tests/RunAll.cmd
@@ -1,6 +1,6 @@
 echo off
 set UT3_OWNER=ut3
-set UT3_OWNER_PASSWORD=ut3
+set UT3_OWNER_PASSWORD=XNtxj8eEgA6X6b6f
 set ORACLE_SID=XE
 if not [%1] == [] (set UT3_OWNER=%1)
 if not [%2] == [] (set UT3_OWNER_PASSWORD=%2)
diff --git a/tests/RunAll.sql b/tests/RunAll.sql
@@ -10,7 +10,8 @@ set longchunksize 1000000
 set serveroutput on size unlimited format truncated
 @@lib/RunVars.sql
 
-@@lib/mystats/mystats start
+@@lib/mystats/mystats_pkg.sql
+exec mystats_pkg.ms_start;
 
 spool RunAll.log
 
@@ -564,7 +565,9 @@ spool coverage.html
 exec ut_output_buffer.lines_to_dbms_output(:html_reporter_id);
 spool off
 
-@@lib/mystats/mystats stop t=1000
+spool stats.log
+exec mystats_pkg.ms_stop(1000);
+spool off
 
 --can be used by CI to check for tests status
 exit :failures_count
diff --git a/tests/lib/mystats/mystats_pkg.sql b/tests/lib/mystats/mystats_pkg.sql

Original file line number	Diff line number	Diff line change
`@@ -223,5 +223,7 @@ create or replace package ut_utils authid definer is`
`223`	`223`
`224`	`224`	`function to_xpath(a_list ut_varchar2_list, a_ancestors varchar2 := '/*/') return varchar2;`
`225`	`225`
	`226`	`+ procedure cleanup_temp_tables;`
	`227`	`+`
`226`	`228`	`end ut_utils;`
`227`	`229`	`/`