From 9c19355b071ca3a9c2971311180e36340c95d35c Mon Sep 17 00:00:00 2001 From: Jacek Date: Fri, 21 Jul 2017 23:53:28 +0100 Subject: [PATCH 1/5] Access to temporary tables improvements. Moved temp tables cleanup into ut_utils (authid definer), so that we can use truncate rather then delete. Refactored ut_coverage and ut_coverage_helper. Changed the RunAll script to use packaged version of mystats. --- source/api/ut_runner.pkb | 11 +--- source/core/coverage/ut_coverage.pkb | 56 +++++++++++++-------- source/core/coverage/ut_coverage_helper.pkb | 30 +++++++---- source/core/coverage/ut_coverage_helper.pks | 4 +- source/core/ut_utils.pkb | 8 +++ source/core/ut_utils.pks | 4 +- tests/RunAll.cmd | 2 +- tests/RunAll.sql | 7 ++- tests/lib/mystats/mystats_pkg.sql | 30 +++++------ 9 files changed, 91 insertions(+), 61 deletions(-) diff --git a/source/api/ut_runner.pkb b/source/api/ut_runner.pkb index 5c723c524..e7113285f 100644 --- a/source/api/ut_runner.pkb +++ b/source/api/ut_runner.pkb @@ -43,13 +43,6 @@ create or replace package body ut_runner is return ut_utils.gc_version; end; - procedure cleanup_temp_tables is - pragma autonomous_transaction; - begin - delete from ut_cursor_data where 1 = 1; - commit; - end; - procedure run( a_paths ut_varchar2_list, a_reporters ut_reporters, a_color_console boolean := false, a_coverage_schemes ut_varchar2_list := null, a_source_file_mappings ut_file_mappings := null, a_test_file_mappings ut_file_mappings := null, @@ -78,11 +71,11 @@ create or replace package body ut_runner is ); l_items_to_run.do_execute(l_listener); - cleanup_temp_tables; + ut_utils.cleanup_temp_tables; ut_output_buffer.close(l_listener.reporters); exception when others then - cleanup_temp_tables; + ut_utils.cleanup_temp_tables; ut_output_buffer.close(l_listener.reporters); dbms_output.put_line(dbms_utility.format_error_backtrace); dbms_output.put_line(dbms_utility.format_error_stack); diff --git a/source/core/coverage/ut_coverage.pkb b/source/core/coverage/ut_coverage.pkb index 4a3b915da..74d344453 100644 --- a/source/core/coverage/ut_coverage.pkb +++ b/source/core/coverage/ut_coverage.pkb @@ -28,7 +28,7 @@ create or replace package body ut_coverage is -- The subquery is optimized by: -- - COALESCE function -> it will execute only for TRIGGERS -- - scalar subquery cache -> it will only execute once for one trigger source code. - function populate_sources_tmp_table(a_coverage_options ut_coverage_options) return varchar2 is + function get_populate_sources_tmp_sql(a_coverage_options ut_coverage_options) return varchar2 is l_result varchar2(32767); l_full_name varchar2(100); begin @@ -98,17 +98,46 @@ create or replace package body ut_coverage is return l_result; end; + function is_tmp_table_populated return boolean is + l_result integer; + begin + select 1 into l_result from ut_coverage_sources_tmp where rownum = 1; + return (l_result = 1); + exception + when no_data_found then + return false; + end; + + procedure populate_tmp_table(a_coverage_options ut_coverage_options, a_skipped_objects ut_object_names) is + pragma autonomous_transaction; + l_schema_names ut_varchar2_rows; + begin + delete from ut_coverage_sources_tmp; + l_schema_names := coalesce(a_coverage_options.schema_names, ut_varchar2_rows(sys_context('USERENV','CURRENT_SCHEMA'))); + if a_coverage_options.file_mappings is not empty then + execute immediate get_populate_sources_tmp_sql(a_coverage_options) using a_coverage_options.file_mappings, a_skipped_objects, a_coverage_options.include_objects; + else + execute immediate get_populate_sources_tmp_sql(a_coverage_options) using l_schema_names, a_skipped_objects, a_coverage_options.include_objects; + end if; + commit; + end; + + /** * Public functions */ procedure coverage_start is begin - ut_coverage_helper.coverage_start('utPLSQL Code coverage run '||ut_utils.to_string(systimestamp)); + if not ut_coverage_helper.is_started() then + ut_coverage_helper.coverage_start('utPLSQL Code coverage run '||ut_utils.to_string(systimestamp)); + end if; end; procedure coverage_start_develop is begin - ut_coverage_helper.coverage_start_develop(); + if not ut_coverage_helper.is_started() then + ut_coverage_helper.coverage_start_develop(); + end if; end; procedure coverage_pause is @@ -132,27 +161,14 @@ create or replace package body ut_coverage is end; function get_coverage_data(a_coverage_options ut_coverage_options) return t_coverage is - - pragma autonomous_transaction; - - type t_coverage_row is record( - name varchar2(500), - line_number integer, - total_occur number(38,0) - ); - type tt_coverage_rows is table of t_coverage_row; l_line_calls ut_coverage_helper.unit_line_calls; l_result t_coverage; l_new_unit t_unit_coverage; l_skipped_objects ut_object_names := ut_object_names(); type t_source_lines is table of binary_integer; - l_source_lines t_source_lines; line_no binary_integer; - l_schema_names ut_varchar2_rows; - l_query varchar2(32767); begin - l_schema_names := coalesce(a_coverage_options.schema_names, ut_varchar2_rows(sys_context('USERENV','CURRENT_SCHEMA'))); if not ut_coverage_helper.is_develop_mode() then --skip all the utplsql framework objects and all the unit test packages that could potentially be reported by coverage. @@ -160,13 +176,9 @@ create or replace package body ut_coverage is end if; --prepare global temp table with sources - delete from ut_coverage_sources_tmp; - if a_coverage_options.file_mappings is not null and a_coverage_options.file_mappings.count > 0 then - execute immediate populate_sources_tmp_table(a_coverage_options) using a_coverage_options.file_mappings, l_skipped_objects, a_coverage_options.include_objects; - else - execute immediate populate_sources_tmp_table(a_coverage_options) using l_schema_names, l_skipped_objects, a_coverage_options.include_objects; + if ut_coverage_helper.is_develop_mode() or not is_tmp_table_populated() then + populate_tmp_table(a_coverage_options, l_skipped_objects); end if; - commit; for src_object in ( select o.owner, o.name, o.full_name, max(o.line) lines_count, diff --git a/source/core/coverage/ut_coverage_helper.pkb b/source/core/coverage/ut_coverage_helper.pkb index 245b33f14..bdc87c2f7 100644 --- a/source/core/coverage/ut_coverage_helper.pkb +++ b/source/core/coverage/ut_coverage_helper.pkb @@ -16,23 +16,30 @@ create or replace package body ut_coverage_helper is limitations under the License. */ - g_coverage_id integer; - g_develop_mode boolean; + g_coverage_id integer; + g_develop_mode boolean := false; + g_is_started boolean := false; - function is_develop_mode return boolean is + function is_develop_mode return boolean is begin return g_develop_mode; end; + function is_started return boolean is + begin + return g_is_started; + end; + procedure coverage_start_internal(a_run_comment varchar2) is begin dbms_profiler.start_profiler(run_comment => a_run_comment, run_number => g_coverage_id); + g_is_started := true; coverage_pause(); end; procedure coverage_start(a_run_comment varchar2) is begin - if g_develop_mode is null then + if not g_is_started then g_develop_mode := false; coverage_start_internal(a_run_comment); end if; @@ -40,8 +47,10 @@ create or replace package body ut_coverage_helper is procedure coverage_start_develop is begin - g_develop_mode := true; - coverage_start_internal('utPLSQL Code coverage run in development MODE '||ut_utils.to_string(systimestamp)); + if not g_is_started then + g_develop_mode := true; + coverage_start_internal('utPLSQL Code coverage run in development MODE '||ut_utils.to_string(systimestamp)); + end if; end; procedure coverage_pause is @@ -59,17 +68,18 @@ create or replace package body ut_coverage_helper is end; procedure coverage_stop is - l_return_code binary_integer; begin if not g_develop_mode then - l_return_code := dbms_profiler.stop_profiler(); + g_is_started := false; + dbms_profiler.stop_profiler(); end if; end; procedure coverage_stop_develop is - l_return_code binary_integer; begin - l_return_code := dbms_profiler.stop_profiler(); + g_develop_mode := false; + g_is_started := false; + dbms_profiler.stop_profiler(); end; function get_raw_coverage_data(a_object_owner varchar2, a_object_name varchar2) return unit_line_calls is diff --git a/source/core/coverage/ut_coverage_helper.pks b/source/core/coverage/ut_coverage_helper.pks index db3fe9c50..921d09682 100644 --- a/source/core/coverage/ut_coverage_helper.pks +++ b/source/core/coverage/ut_coverage_helper.pks @@ -17,11 +17,13 @@ create or replace package ut_coverage_helper authid definer is */ --table of line calls indexed by line number - -- table is sparse!!! + --!!! this table is sparse!!! type unit_line_calls is table of number(38,0) index by binary_integer; function is_develop_mode return boolean; + function is_started return boolean; + procedure coverage_start(a_run_comment varchar2); /* diff --git a/source/core/ut_utils.pkb b/source/core/ut_utils.pkb index a0c4eca9e..84d5a05e3 100644 --- a/source/core/ut_utils.pkb +++ b/source/core/ut_utils.pkb @@ -341,5 +341,13 @@ create or replace package body ut_utils is return l_result; end; + procedure cleanup_temp_tables is + pragma autonomous_transaction; + begin + execute immediate 'truncate table ut_cursor_data'; + execute immediate 'truncate table ut_coverage_sources_tmp$'; + commit; + end; + end ut_utils; / diff --git a/source/core/ut_utils.pks b/source/core/ut_utils.pks index 308677bc7..d84865949 100644 --- a/source/core/ut_utils.pks +++ b/source/core/ut_utils.pks @@ -79,7 +79,7 @@ create or replace package ut_utils authid definer is ex_invalid_rep_event_name exception; gc_invalid_rep_event_name constant pls_integer := -20211; pragma exception_init(ex_invalid_rep_event_name, -20211); - + -- Any of tests failed ex_some_tests_failed exception; gc_some_tests_failed constant pls_integer := -20213; @@ -219,5 +219,7 @@ create or replace package ut_utils authid definer is function convert_collection(a_collection ut_varchar2_list) return ut_varchar2_rows; + procedure cleanup_temp_tables; + end ut_utils; / diff --git a/tests/RunAll.cmd b/tests/RunAll.cmd index c0a3eef1c..041d2be96 100644 --- a/tests/RunAll.cmd +++ b/tests/RunAll.cmd @@ -1,6 +1,6 @@ echo off set UT3_OWNER=ut3 -set UT3_OWNER_PASSWORD=ut3 +set UT3_OWNER_PASSWORD=XNtxj8eEgA6X6b6f set ORACLE_SID=XE if not [%1] == [] (set UT3_OWNER=%1) if not [%2] == [] (set UT3_OWNER_PASSWORD=%2) diff --git a/tests/RunAll.sql b/tests/RunAll.sql index 701b32fd9..203422e41 100644 --- a/tests/RunAll.sql +++ b/tests/RunAll.sql @@ -10,7 +10,8 @@ set longchunksize 1000000 set serveroutput on size unlimited format truncated @@lib/RunVars.sql -@@lib/mystats/mystats start +@@lib/mystats/mystats_pkg.sql +exec mystats_pkg.ms_start; spool RunAll.log @@ -557,7 +558,9 @@ spool coverage.html exec ut_output_buffer.lines_to_dbms_output(:html_reporter_id); spool off -@@lib/mystats/mystats stop t=1000 +spool stats.log +exec mystats_pkg.ms_stop(1000); +spool off --can be used by CI to check for tests status exit :failures_count diff --git a/tests/lib/mystats/mystats_pkg.sql b/tests/lib/mystats/mystats_pkg.sql index e1938f182..7e9298c5d 100644 --- a/tests/lib/mystats/mystats_pkg.sql +++ b/tests/lib/mystats/mystats_pkg.sql @@ -13,10 +13,10 @@ create or replace package mystats_pkg authid current_user as || || Author: Adrian Billington || www.oracle-developer.net - || (c) oracle-developer.net + || (c) oracle-developer.net || || Description: PL/SQL-only version of Jonathan Lewis's SNAP_MY_STATS package. - || This package is used to output the resource usage as recorded + || This package is used to output the resource usage as recorded || in V$MYSTAT and V$LATCH. || || Key Differences @@ -50,7 +50,7 @@ create or replace package mystats_pkg authid current_user as || exec mystats_pkg.ms_stop; || || 2. Output statistics with delta values >= 1,000 - || ------------------------------------------------------------- + || ------------------------------------------------------------- || exec mystats_pkg.ms_start; || ---- || exec mystats_pkg.ms_stop(1000); @@ -61,21 +61,21 @@ create or replace package mystats_pkg authid current_user as || ---- || exec mystats_pkg.ms_stop(mystats_pkg.statname_ntt('redo size', 'user commits')); || - || 4. Output statistics for those containing the word 'memory' + || 4. Output statistics for those containing the word 'memory' || ----------------------------------------------------------- || exec mystats_pkg.ms_start; || ---- || exec mystats_pkg.ms_stop('memory'); || || Notes: 1. Serveroutput must be on (and set higher than default); - || + || || 2. See http://www.jlcomp.demon.co.uk/snapshot.html for original || version. || - || 3. A free-standing, SQL*Plus-script version of MyStats is also + || 3. A free-standing, SQL*Plus-script version of MyStats is also || available. The script version works without creating any || database objects. - || + || || Disclaimer: http://www.oracle-developer.net/disclaimer.php || || ---------------------------------------------------------------------------- @@ -183,7 +183,7 @@ create or replace package body mystats_pkg as union all select 'LATCH' , name - , gets + , gets from v$latch union all select 'TIME' @@ -221,7 +221,7 @@ create or replace package body mystats_pkg as /* || Downside of using associative arrays is that we have to sort - || the output. So here's a couple of types and a variable to enable us + || the output. So here's a couple of types and a variable to enable us || to do that... */ type aat_mystats_output is table of st_output @@ -315,13 +315,13 @@ create or replace package body mystats_pkg as v_value := ga_mystats(c_run2)(v_name).value - ga_mystats(c_run1)(v_name).value; /* - || If it's greater than the threshold or a statistic we are interested in, + || If it's greater than the threshold or a statistic we are interested in, || then output it. The downside of using purely associative arrays is that || we don't have any easy way of sorting. So we have to do it ourselves... */ if (p_threshold is not null and abs(v_value) >= p_threshold) or (p_statnames is not empty and v_name member of p_statnames) - or (p_statname_like is not null and v_name like '%'||p_statname_like||'%') + or (p_statname_like is not null and v_name like '%'||p_statname_like||'%') then /* || Fix for bug 1713403. If redo goes over 2Gb then it is reported as a negative @@ -334,7 +334,7 @@ create or replace package body mystats_pkg as sort(v_type, v_name, v_value); end if; end if; - + /* || Next statname please... */ @@ -402,7 +402,7 @@ create or replace package body mystats_pkg as then ms_report(p_statnames => p_statnames); when p_statname_like is not null then ms_report(p_statname_like => p_statname_like); - else ms_report; + else ms_report; end case; ms_reset; else @@ -438,5 +438,5 @@ create or replace package body mystats_pkg as end mystats_pkg; / -create or replace public synonym mystats_pkg for mystats_pkg; -grant execute on mystats_pkg to public; +-- create or replace public synonym mystats_pkg for mystats_pkg; +-- grant execute on mystats_pkg to public; From 78f37904d75052fc43e2ac65782b7022fde49afe Mon Sep 17 00:00:00 2001 From: Jacek Date: Sat, 22 Jul 2017 01:36:12 +0100 Subject: [PATCH 2/5] dbms_profiler issue workaround --- source/core/coverage/ut_coverage_helper.pkb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/source/core/coverage/ut_coverage_helper.pkb b/source/core/coverage/ut_coverage_helper.pkb index bdc87c2f7..a751d9ecf 100644 --- a/source/core/coverage/ut_coverage_helper.pkb +++ b/source/core/coverage/ut_coverage_helper.pkb @@ -91,7 +91,13 @@ create or replace package body ut_coverage_helper is l_tmp_data coverage_rows; l_results unit_line_calls; begin - select d.line#, d.total_occur + select d.line#, + -- This transformation addresses two issues: + -- 1. dbms_profiler shows multiple unit_number for single code unit; + -- to address this, we take a sum od all units by name + -- 2. some lines show 0 total_occur while they were executed (time > 0) + -- in this case we show 1 to indicate that there was execution even if we don't know how many there were + case when sum(d.total_occur) = 0 and sum(d.total_time) > 0 then 1 else sum(d.total_occur) end total_occur bulk collect into l_tmp_data from plsql_profiler_units u join plsql_profiler_data d @@ -101,7 +107,8 @@ create or replace package body ut_coverage_helper is and u.unit_owner = a_object_owner and u.unit_name = a_object_name --exclude specification - and u.unit_type not in ('PACKAGE SPEC', 'TYPE SPEC', 'ANONYMOUS BLOCK'); + and u.unit_type not in ('PACKAGE SPEC', 'TYPE SPEC', 'ANONYMOUS BLOCK') + group by d.line#; for i in 1 .. l_tmp_data.count loop l_results(l_tmp_data(i).line) := l_tmp_data(i).calls; end loop; From af5ae33968580143d65dc920d45137f26f4722ea Mon Sep 17 00:00:00 2001 From: Jacek Date: Sat, 22 Jul 2017 01:39:33 +0100 Subject: [PATCH 3/5] Removing commit after autonomous was removed. --- source/core/coverage/ut_coverage.pkb | 1 - 1 file changed, 1 deletion(-) diff --git a/source/core/coverage/ut_coverage.pkb b/source/core/coverage/ut_coverage.pkb index 74d344453..8e2839c7b 100644 --- a/source/core/coverage/ut_coverage.pkb +++ b/source/core/coverage/ut_coverage.pkb @@ -236,7 +236,6 @@ create or replace package body ut_coverage is end loop; - commit; return l_result; end get_coverage_data; From 2185cecdd01642ee76f5dc525cfa21a1697dbead Mon Sep 17 00:00:00 2001 From: Jacek Date: Sat, 22 Jul 2017 10:45:03 +0100 Subject: [PATCH 4/5] Addressing review comments. --- source/core/coverage/ut_coverage.pkb | 8 ++------ source/core/coverage/ut_coverage_helper.pkb | 4 ++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/source/core/coverage/ut_coverage.pkb b/source/core/coverage/ut_coverage.pkb index 8e2839c7b..24d26bbd1 100644 --- a/source/core/coverage/ut_coverage.pkb +++ b/source/core/coverage/ut_coverage.pkb @@ -128,16 +128,12 @@ create or replace package body ut_coverage is */ procedure coverage_start is begin - if not ut_coverage_helper.is_started() then - ut_coverage_helper.coverage_start('utPLSQL Code coverage run '||ut_utils.to_string(systimestamp)); - end if; + ut_coverage_helper.coverage_start('utPLSQL Code coverage run '||ut_utils.to_string(systimestamp)); end; procedure coverage_start_develop is begin - if not ut_coverage_helper.is_started() then - ut_coverage_helper.coverage_start_develop(); - end if; + ut_coverage_helper.coverage_start_develop(); end; procedure coverage_pause is diff --git a/source/core/coverage/ut_coverage_helper.pkb b/source/core/coverage/ut_coverage_helper.pkb index a751d9ecf..746f7f601 100644 --- a/source/core/coverage/ut_coverage_helper.pkb +++ b/source/core/coverage/ut_coverage_helper.pkb @@ -17,8 +17,8 @@ create or replace package body ut_coverage_helper is */ g_coverage_id integer; - g_develop_mode boolean := false; - g_is_started boolean := false; + g_develop_mode boolean not null := false; + g_is_started boolean not null := false; function is_develop_mode return boolean is begin From f6e8271b2c20020d23a599fb4a0e76d1568b7fc0 Mon Sep 17 00:00:00 2001 From: Jacek Date: Mon, 24 Jul 2017 23:49:27 +0100 Subject: [PATCH 5/5] Removed unused code. Changed order of processing in if statement to make sure query in `is_tmp_table_populated` is executed when running unit tests. --- source/core/coverage/ut_coverage.pkb | 2 +- source/core/coverage/ut_coverage_helper.pkb | 5 ----- source/core/coverage/ut_coverage_helper.pks | 2 -- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/source/core/coverage/ut_coverage.pkb b/source/core/coverage/ut_coverage.pkb index 24d26bbd1..b6a832038 100644 --- a/source/core/coverage/ut_coverage.pkb +++ b/source/core/coverage/ut_coverage.pkb @@ -172,7 +172,7 @@ create or replace package body ut_coverage is end if; --prepare global temp table with sources - if ut_coverage_helper.is_develop_mode() or not is_tmp_table_populated() then + if not is_tmp_table_populated() or ut_coverage_helper.is_develop_mode() then populate_tmp_table(a_coverage_options, l_skipped_objects); end if; diff --git a/source/core/coverage/ut_coverage_helper.pkb b/source/core/coverage/ut_coverage_helper.pkb index 746f7f601..b2434f592 100644 --- a/source/core/coverage/ut_coverage_helper.pkb +++ b/source/core/coverage/ut_coverage_helper.pkb @@ -25,11 +25,6 @@ create or replace package body ut_coverage_helper is return g_develop_mode; end; - function is_started return boolean is - begin - return g_is_started; - end; - procedure coverage_start_internal(a_run_comment varchar2) is begin dbms_profiler.start_profiler(run_comment => a_run_comment, run_number => g_coverage_id); diff --git a/source/core/coverage/ut_coverage_helper.pks b/source/core/coverage/ut_coverage_helper.pks index 921d09682..cc966c2b0 100644 --- a/source/core/coverage/ut_coverage_helper.pks +++ b/source/core/coverage/ut_coverage_helper.pks @@ -22,8 +22,6 @@ create or replace package ut_coverage_helper authid definer is function is_develop_mode return boolean; - function is_started return boolean; - procedure coverage_start(a_run_comment varchar2); /*