Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit da6ee70

Browse files
jgebaljgebal
committed
Refactored output buffers to have better segregation of responsibilities.
Changed behavior of output buffer `get_lines` procedure to stop immediately after consuming all produced data if producer has stopped. Changed behavior of output buffer `get_lines` procedure not to timeout after 4 hours, if producer is still running.
1 parent f72db1e commit da6ee70

14 files changed

Lines changed: 277 additions & 258 deletions

‎docs/userguide/install.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ The headless scripts accept three optional parameters that define:
121121
The scripts need to be executed by `SYSDBA`, in order to grant access to `DBMS_LOCK` and `DBMS_CRYPTO` system packages.
122122

123123
!!! warning "Important"
124-
- Grant on `DBMS_LOCK` is required only for installation on Oracle versions below 18c. For versions 18c and above, utPLSQL uses `DBMS_SESSION.SLEEP` so access to `DBMS_LOCK` package is no longer needed.<br>
124+
- `DBMS_LOCK` is required for session synchronization between main session and session consuming realtime reports.<br>
125125
- The user performing the installation must have the `ADMINISTER DATABASE TRIGGER` privilege. This is required for installation of trigger that is responsible for parsing annotations at at compile-time of a package.<br>
126126
- When installed with DDL trigger, utPLSQL will not be registering unit tests for any of oracle-maintained schemas.<br>
127127
- For Oracle 11g following users are excluded:<br>

‎source/core/coverage/ut_coverage_reporter_base.tpb‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,6 @@ create or replace type body ut_coverage_reporter_base is
9292
ut_coverage_helper.cleanup_tmp_table();
9393
(l_reporter as ut_output_reporter_base).before_calling_run(null);
9494
l_reporter.after_calling_run( ut_run( a_coverage_options => a_coverage_options, a_client_character_set => a_client_character_set ) );
95-
l_reporter.on_finalize(null);
9695
for i in (select /*+ no_parallel */ x.text from table(l_reporter.get_lines(1, 1)) x ) loop
9796
pipe row (i.text);
9897
end loop;
@@ -106,7 +105,6 @@ create or replace type body ut_coverage_reporter_base is
106105
ut_coverage_helper.cleanup_tmp_table();
107106
(l_reporter as ut_output_reporter_base).before_calling_run(null);
108107
l_reporter.after_calling_run( ut_run( a_coverage_options => a_coverage_options, a_client_character_set => a_client_character_set ) );
109-
l_reporter.on_finalize(null);
110108
open l_result for select /*+ no_parallel */ x.text from table(l_reporter.get_lines(1, 1)) x;
111109
return l_result;
112110
end;

‎source/core/output_buffers/ut_output_buffer_base.tpb‎

Lines changed: 129 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,18 +24,143 @@ create or replace type body ut_output_buffer_base is
2424
self.self_type := coalesce(a_self_type,self.self_type);
2525
self.output_id := coalesce(a_output_id, self.output_id, sys_guid());
2626
self.start_date := coalesce(self.start_date, sysdate);
27-
self.last_message_id := 0;
27+
self.last_write_message_id := 0;
2828
select /*+ no_parallel */ count(*) into l_exists from ut_output_buffer_info_tmp where output_id = self.output_id;
2929
if ( l_exists > 0 ) then
3030
update /*+ no_parallel */ ut_output_buffer_info_tmp set start_date = self.start_date where output_id = self.output_id;
3131
else
3232
insert /*+ no_parallel */ into ut_output_buffer_info_tmp(output_id, start_date) values (self.output_id, self.start_date);
3333
end if;
3434
commit;
35+
dbms_lock.allocate_unique( self.output_id, self.lock_handle);
3536
self.is_closed := 0;
3637
end;
3738

38-
member function get_lines_cursor(a_initial_timeout natural := null, a_timeout_sec natural := null) return sys_refcursor is
39+
member procedure lock_buffer(a_timeout_sec number := null) is
40+
l_status integer;
41+
begin
42+
l_status := dbms_lock.request( self.lock_handle, dbms_lock.x_mode, 5, false );
43+
if l_status != 0 then
44+
raise_application_error(-20000, 'Cannot allocate lock for output buffer of reporter. lock request status = '||l_status||', lock handle = '||self.lock_handle||', self.output_id ='||self.output_id);
45+
end if;
46+
end;
47+
48+
member procedure close(self in out nocopy ut_output_buffer_base) is
49+
l_status integer;
50+
begin
51+
l_status := dbms_lock.release( self.lock_handle );
52+
if l_status != 0 then
53+
raise_application_error(-20000, 'Cannot release lock for output buffer of reporter. Lock_handle = '||self.lock_handle||' status = '||l_status);
54+
end if;
55+
self.is_closed := 1;
56+
end;
57+
58+
59+
member procedure remove_buffer_info(self in ut_output_buffer_base) is
60+
pragma autonomous_transaction;
61+
begin
62+
delete from ut_output_buffer_info_tmp a
63+
where a.output_id = self.output_id;
64+
commit;
65+
end;
66+
67+
member function get_lines(a_initial_timeout number := null, a_timeout_sec number := null) return ut_output_data_rows pipelined is
68+
lc_init_wait_sec constant number := coalesce(a_initial_timeout, 10 );
69+
lc_100_milisec constant number(1,1) := 0.1; --sleep for 100 ms between checks
70+
lc_500_milisec constant number(3,1) := 0.5; --sleep for 1 s when waiting long
71+
lc_1_second constant number(3,1) := 1;
72+
l_buffer_rowids ut_varchar2_rows;
73+
l_buffer_data ut_output_data_rows;
74+
l_finished_flags ut_integer_list;
75+
l_last_read_message_id integer;
76+
l_already_waited_sec number(10,2) := 0;
77+
l_data_finished boolean := false;
78+
l_finished boolean := false;
79+
l_sleep_time number(2,1) := lc_100_milisec;
80+
l_lock_status integer;
81+
l_producer_started boolean := false;
82+
l_producer_finished boolean := false;
83+
function get_lock_status return integer is
84+
l_result integer;
85+
l_release_status integer;
86+
begin
87+
l_result := dbms_lock.request( self.lock_handle, dbms_lock.s_mode, 0, false );
88+
if l_result = 0 then
89+
l_release_status := dbms_lock.release( self.lock_handle );
90+
end if;
91+
return l_result;
92+
end;
93+
begin
94+
while not l_finished loop
95+
96+
--check if the lock is still there on output - if yes, the main session is still running and so don't stop
97+
l_lock_status := get_lock_status();
98+
get_data_from_buffer_table( l_last_read_message_id, l_buffer_data, l_buffer_rowids, l_finished_flags );
99+
100+
--nothing fetched from output, wait and try again
101+
if l_buffer_data.count = 0 then
102+
103+
dbms_lock.sleep(l_sleep_time);
104+
l_already_waited_sec := l_already_waited_sec + l_sleep_time;
105+
106+
-- if waited more than lc_1_second seconds then increase wait period to minimize the CPU usage.
107+
if l_already_waited_sec >= lc_1_second then
108+
l_sleep_time := lc_500_milisec;
109+
end if;
110+
111+
else
112+
113+
l_already_waited_sec := 0;
114+
l_sleep_time := lc_100_milisec;
115+
116+
for i in 1 .. l_buffer_data.count loop
117+
if l_buffer_data(i).text is not null then
118+
pipe row( l_buffer_data(i) );
119+
elsif l_finished_flags(i) = 1 then
120+
l_data_finished := true;
121+
exit;
122+
end if;
123+
end loop;
124+
125+
remove_read_data(l_buffer_rowids);
126+
127+
end if;
128+
l_producer_started := (l_lock_status <> 0 or l_buffer_data.count > 0) or l_producer_started;
129+
l_producer_finished := (l_producer_started and l_lock_status = 0 and l_buffer_data.count = 0) or l_producer_finished;
130+
131+
if not l_producer_started and l_already_waited_sec >= lc_init_wait_sec then
132+
133+
if lc_init_wait_sec > 0 then
134+
self.remove_buffer_info();
135+
raise_application_error(
136+
ut_utils.gc_out_buffer_timeout,
137+
'Timeout occurred while waiting for report data producer to start. Waited for: '||ut_utils.to_string( l_already_waited_sec )||' seconds.'
138+
);
139+
else
140+
l_finished := true;
141+
end if;
142+
143+
elsif not l_producer_finished and a_timeout_sec is not null and l_already_waited_sec >= a_timeout_sec then
144+
145+
if a_timeout_sec > 0 then
146+
self.remove_buffer_info();
147+
raise_application_error(
148+
ut_utils.gc_out_buffer_timeout,
149+
'Timeout occurred while waiting for more data from producer. Waited for: '||ut_utils.to_string( l_already_waited_sec )||' seconds.'
150+
);
151+
else
152+
l_finished := true;
153+
end if;
154+
155+
elsif (l_data_finished or l_producer_finished) then
156+
l_finished := true;
157+
end if;
158+
end loop;
159+
self.remove_buffer_info();
160+
return;
161+
end;
162+
163+
member function get_lines_cursor(a_initial_timeout number := null, a_timeout_sec number := null) return sys_refcursor is
39164
l_lines sys_refcursor;
40165
begin
41166
open l_lines for
@@ -44,7 +169,7 @@ create or replace type body ut_output_buffer_base is
44169
return l_lines;
45170
end;
46171

47-
member procedure lines_to_dbms_output(self in ut_output_buffer_base, a_initial_timeout natural := null, a_timeout_sec natural := null) is
172+
member procedure lines_to_dbms_output(self in ut_output_buffer_base, a_initial_timeout number := null, a_timeout_sec number := null) is
48173
l_data sys_refcursor;
49174
l_clob clob;
50175
l_item_type varchar2(32767);
@@ -63,7 +188,7 @@ create or replace type body ut_output_buffer_base is
63188
end;
64189

65190
member procedure cleanup_buffer(self in ut_output_buffer_base, a_retention_time_sec natural := null) is
66-
gc_buffer_retention_sec constant naturaln := coalesce(a_retention_time_sec, 60 * 60 * 24); -- 24 hours
191+
gc_buffer_retention_sec constant naturaln := coalesce(a_retention_time_sec, 60 * 60 * 24 * 5); -- 5 days
67192
l_retention_days number := gc_buffer_retention_sec / (60 * 60 * 24);
68193
l_max_retention_date date := sysdate - l_retention_days;
69194
pragma autonomous_transaction;

‎source/core/output_buffers/ut_output_buffer_base.tps‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,27 @@ create or replace type ut_output_buffer_base force authid definer as object(
1919
output_id raw(32),
2020
is_closed number(1,0),
2121
start_date date,
22-
last_message_id number(38,0),
22+
last_write_message_id number(38,0),
23+
lock_handle varchar2(30 byte),
2324
self_type varchar2(250 byte),
2425
member procedure init(self in out nocopy ut_output_buffer_base, a_output_id raw := null, a_self_type varchar2 := null),
25-
member function get_lines_cursor(a_initial_timeout natural := null, a_timeout_sec natural := null) return sys_refcursor,
26-
member procedure lines_to_dbms_output(self in ut_output_buffer_base, a_initial_timeout natural := null, a_timeout_sec natural := null),
26+
member procedure lock_buffer(a_timeout_sec number := null),
27+
member function get_lines_cursor(a_initial_timeout number := null, a_timeout_sec number := null) return sys_refcursor,
28+
member procedure lines_to_dbms_output(self in ut_output_buffer_base, a_initial_timeout number := null, a_timeout_sec number := null),
2729
member procedure cleanup_buffer(self in ut_output_buffer_base, a_retention_time_sec natural := null),
28-
not instantiable member procedure close(self in out nocopy ut_output_buffer_base),
30+
member procedure remove_buffer_info(self in ut_output_buffer_base),
31+
not instantiable member procedure get_data_from_buffer_table(
32+
self in ut_output_buffer_base,
33+
a_last_read_message_id in out nocopy integer,
34+
a_buffer_data out nocopy ut_output_data_rows,
35+
a_buffer_rowids out nocopy ut_varchar2_rows,
36+
a_finished_flags out nocopy ut_integer_list
37+
),
38+
member procedure close(self in out nocopy ut_output_buffer_base),
39+
not instantiable member procedure remove_read_data(self in ut_output_buffer_base, a_buffer_rowids ut_varchar2_rows),
2940
not instantiable member procedure send_line(self in out nocopy ut_output_buffer_base, a_text varchar2, a_item_type varchar2 := null),
3041
not instantiable member procedure send_lines(self in out nocopy ut_output_buffer_base, a_text_list ut_varchar2_rows, a_item_type varchar2 := null),
3142
not instantiable member procedure send_clob(self in out nocopy ut_output_buffer_base, a_text clob, a_item_type varchar2 := null),
32-
not instantiable member function get_lines(a_initial_timeout natural := null, a_timeout_sec natural := null) return ut_output_data_rows pipelined
43+
member function get_lines(a_initial_timeout number := null, a_timeout_sec number := null) return ut_output_data_rows pipelined
3344
) not final not instantiable
3445
/

‎source/core/output_buffers/ut_output_clob_table_buffer.tpb‎

Lines changed: 34 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -22,23 +22,13 @@ create or replace type body ut_output_clob_table_buffer is
2222
return;
2323
end;
2424

25-
overriding member procedure close(self in out nocopy ut_output_clob_table_buffer) is
26-
pragma autonomous_transaction;
27-
begin
28-
self.last_message_id := self.last_message_id + 1;
29-
insert /*+ no_parallel */ into ut_output_clob_buffer_tmp(output_id, message_id, is_finished)
30-
values (self.output_id, self.last_message_id, 1);
31-
commit;
32-
self.is_closed := 1;
33-
end;
34-
3525
overriding member procedure send_line(self in out nocopy ut_output_clob_table_buffer, a_text varchar2, a_item_type varchar2 := null) is
3626
pragma autonomous_transaction;
3727
begin
3828
if a_text is not null or a_item_type is not null then
39-
self.last_message_id := self.last_message_id + 1;
29+
self.last_write_message_id := self.last_write_message_id + 1;
4030
insert /*+ no_parallel */ into ut_output_clob_buffer_tmp(output_id, message_id, text, item_type)
41-
values (self.output_id, self.last_message_id, a_text, a_item_type);
31+
values (self.output_id, self.last_write_message_id, a_text, a_item_type);
4232
end if;
4333
commit;
4434
end;
@@ -47,110 +37,54 @@ create or replace type body ut_output_clob_table_buffer is
4737
pragma autonomous_transaction;
4838
begin
4939
insert /*+ no_parallel */ into ut_output_clob_buffer_tmp(output_id, message_id, text, item_type)
50-
select /*+ no_parallel */ self.output_id, self.last_message_id + rownum, t.column_value, a_item_type
40+
select /*+ no_parallel */ self.output_id, self.last_write_message_id + rownum, t.column_value, a_item_type
5141
from table(a_text_list) t
5242
where t.column_value is not null or a_item_type is not null;
53-
self.last_message_id := self.last_message_id + SQL%rowcount;
43+
self.last_write_message_id := self.last_write_message_id + SQL%rowcount;
5444
commit;
5545
end;
5646

5747
overriding member procedure send_clob(self in out nocopy ut_output_clob_table_buffer, a_text clob, a_item_type varchar2 := null) is
5848
pragma autonomous_transaction;
5949
begin
6050
if a_text is not null and a_text != empty_clob() or a_item_type is not null then
61-
self.last_message_id := self.last_message_id + 1;
51+
self.last_write_message_id := self.last_write_message_id + 1;
6252
insert /*+ no_parallel */ into ut_output_clob_buffer_tmp(output_id, message_id, text, item_type)
63-
values (self.output_id, self.last_message_id, a_text, a_item_type);
53+
values (self.output_id, self.last_write_message_id, a_text, a_item_type);
6454
end if;
6555
commit;
6656
end;
6757

68-
overriding member function get_lines(a_initial_timeout natural := null, a_timeout_sec natural := null) return ut_output_data_rows pipelined is
69-
type t_rowid_tab is table of urowid;
70-
l_message_rowids t_rowid_tab;
71-
l_buffer_data ut_output_data_rows;
72-
l_finished_flags ut_integer_list;
73-
l_already_waited_for number(10,2) := 0;
74-
l_finished boolean := false;
75-
lc_init_wait_sec constant naturaln := coalesce(a_initial_timeout, 60 ); -- 1 minute
76-
lc_max_wait_sec constant naturaln := coalesce(a_timeout_sec, 60 * 60 * 4); -- 4 hours
77-
l_wait_for integer := lc_init_wait_sec;
78-
lc_short_sleep_time constant number(1,1) := 0.1; --sleep for 100 ms between checks
79-
lc_long_sleep_time constant number(1) := 1; --sleep for 1 s when waiting long
80-
lc_long_wait_time constant number(1) := 1; --waiting more than 1 sec
81-
l_sleep_time number(2,1) := lc_short_sleep_time;
82-
lc_bulk_limit constant integer := 5000;
83-
l_max_message_id integer := lc_bulk_limit;
84-
85-
procedure remove_read_data(a_message_rowids t_rowid_tab) is
86-
pragma autonomous_transaction;
87-
begin
88-
forall i in 1 .. a_message_rowids.count
89-
delete from ut_output_clob_buffer_tmp a
90-
where rowid = a_message_rowids(i);
91-
commit;
92-
end;
58+
overriding member procedure get_data_from_buffer_table(
59+
self in ut_output_clob_table_buffer,
60+
a_last_read_message_id in out nocopy integer,
61+
a_buffer_data out nocopy ut_output_data_rows,
62+
a_buffer_rowids out nocopy ut_varchar2_rows,
63+
a_finished_flags out nocopy ut_integer_list
64+
) is
65+
lc_bulk_limit constant integer := 5000;
66+
begin
67+
a_last_read_message_id := coalesce(a_last_read_message_id, 0);
68+
with ordered_buffer as (
69+
select /*+ no_parallel index(a) */ ut_output_data_row(a.text, a.item_type), rowidtochar(a.rowid), is_finished
70+
from ut_output_clob_buffer_tmp a
71+
where a.output_id = self.output_id
72+
and a.message_id <= a_last_read_message_id + lc_bulk_limit
73+
order by a.message_id
74+
)
75+
select /*+ no_parallel */ b.*
76+
bulk collect into a_buffer_data, a_buffer_rowids, a_finished_flags
77+
from ordered_buffer b;
78+
a_last_read_message_id := a_last_read_message_id + a_finished_flags.count;
79+
end;
9380

94-
procedure remove_buffer_info is
81+
overriding member procedure remove_read_data(self in ut_output_clob_table_buffer, a_buffer_rowids ut_varchar2_rows) is
9582
pragma autonomous_transaction;
96-
begin
97-
delete from ut_output_buffer_info_tmp a
98-
where a.output_id = self.output_id;
99-
commit;
100-
end;
101-
102-
begin
103-
while not l_finished loop
104-
with ordered_buffer as (
105-
select /*+ no_parallel index(a) */ a.rowid, ut_output_data_row(a.text, a.item_type), is_finished
106-
from ut_output_clob_buffer_tmp a
107-
where a.output_id = self.output_id
108-
and a.message_id <= l_max_message_id
109-
order by a.message_id
110-
)
111-
select /*+ no_parallel */ b.*
112-
bulk collect into l_message_rowids, l_buffer_data, l_finished_flags
113-
from ordered_buffer b;
114-
115-
--nothing fetched from output, wait and try again
116-
if l_buffer_data.count = 0 then
117-
$if dbms_db_version.version >= 18 $then
118-
dbms_session.sleep(l_sleep_time);
119-
$else
120-
dbms_lock.sleep(l_sleep_time);
121-
$end
122-
l_already_waited_for := l_already_waited_for + l_sleep_time;
123-
if l_already_waited_for > lc_long_wait_time then
124-
l_sleep_time := lc_long_sleep_time;
125-
end if;
126-
else
127-
--reset wait time
128-
-- we wait lc_max_wait_sec for new message
129-
l_wait_for := lc_max_wait_sec;
130-
l_already_waited_for := 0;
131-
l_sleep_time := lc_short_sleep_time;
132-
for i in 1 .. l_buffer_data.count loop
133-
if l_buffer_data(i).text is not null then
134-
pipe row(l_buffer_data(i));
135-
elsif l_finished_flags(i) = 1 then
136-
l_finished := true;
137-
exit;
138-
end if;
139-
end loop;
140-
remove_read_data(l_message_rowids);
141-
l_max_message_id := l_max_message_id + lc_bulk_limit;
142-
end if;
143-
if l_finished or l_already_waited_for >= l_wait_for then
144-
remove_buffer_info();
145-
if l_already_waited_for > 0 and l_already_waited_for >= l_wait_for then
146-
raise_application_error(
147-
ut_utils.gc_out_buffer_timeout,
148-
'Timeout occurred while waiting for output data. Waited for: '||l_already_waited_for||' seconds.'
149-
);
150-
end if;
151-
end if;
152-
end loop;
153-
return;
83+
begin
84+
forall i in 1 .. a_buffer_rowids.count
85+
delete from ut_output_clob_buffer_tmp a
86+
where rowid = chartorowid(a_buffer_rowids(i));
87+
commit;
15488
end;
15589

15690
end;

0 commit comments

Comments
 (0)