Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit aafca84

Browse files
jgebaljgebal
committed
Refactored annotation cache.
* Added ability to build cache in `ut_annotation_manager.rebuild_annotation_cache` * Added ability to purge cache in `ut_annotation_cache_manager.purge_cache`
1 parent 6dcaae9 commit aafca84

16 files changed

Lines changed: 590 additions & 120 deletions

‎source/core/annotations/ut_annotation_cache_manager.pkb‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ create or replace package body ut_annotation_cache_manager as
5353
end;
5454

5555

56-
procedure cleanup_cache(a_objects ut_annotation_cached_objects) is
56+
procedure cleanup_cache(a_objects ut_annotation_objs_cache_info) is
5757
pragma autonomous_transaction;
5858
begin
5959

@@ -79,7 +79,7 @@ create or replace package body ut_annotation_cache_manager as
7979
commit;
8080
end;
8181

82-
function get_annotations_for_objects(a_cached_objects ut_annotation_cached_objects) return sys_refcursor is
82+
function get_annotations_for_objects(a_cached_objects ut_annotation_objs_cache_info) return sys_refcursor is
8383
l_results sys_refcursor;
8484
begin
8585
open l_results for
@@ -94,9 +94,34 @@ create or replace package body ut_annotation_cache_manager as
9494
)
9595
)
9696
from table(a_cached_objects) o
97-
join ut_annotation_cache c on o.cache_id = c.cache_id
97+
join ut_annotation_cache_info i
98+
on o.object_owner = i.object_owner and o.object_name = i.object_name and o.object_type = i.object_type
99+
join ut_annotation_cache c on i.cache_id = c.cache_id
98100
group by o.object_owner, o.object_name, o.object_type;
99101
return l_results;
100102
end;
103+
104+
procedure purge_cache(a_object_owner varchar2, a_object_type varchar2) is
105+
pragma autonomous_transaction;
106+
begin
107+
execute immediate '
108+
delete from ut_annotation_cache c
109+
where c.cache_id
110+
in (select i.cache_id
111+
from ut_annotation_cache_info i
112+
where 1 = 1
113+
and '||case when a_object_owner is null then ':a_object_owner is null' else 'object_owner = :a_object_owner' end || '
114+
and '||case when a_object_type is null then ':a_object_type is null' else 'object_type = :a_object_type' end || '
115+
)'
116+
using a_object_owner, a_object_type;
117+
execute immediate '
118+
delete from ut_annotation_cache_info i
119+
where 1 = 1
120+
and '||case when a_object_owner is null then ':a_object_owner is null' else 'object_owner = :a_object_owner' end || '
121+
and '||case when a_object_type is null then ':a_object_type is null' else 'object_type = :a_object_type' end
122+
using a_object_owner, a_object_type;
123+
commit;
124+
end;
125+
101126
end ut_annotation_cache_manager;
102127
/

‎source/core/annotations/ut_annotation_cache_manager.pks‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,38 @@ create or replace package ut_annotation_cache_manager authid definer as
1616
limitations under the License.
1717
*/
1818

19+
/**
20+
* Populates cache with information about object and it's annotations
21+
* Cache information for individual object is modified by this code
22+
* We do not pass a collection here to avoid excessive memory usage
23+
* when dealing with large number of objects
24+
*
25+
* @param a_object a `ut_annotated_object` containing object name, type, owner and `ut_annotations`
26+
*/
1927
procedure update_cache(a_object ut_annotated_object);
2028

21-
function get_annotations_for_objects(a_cached_objects ut_annotation_cached_objects) return sys_refcursor;
29+
/**
30+
* Returns a ref_cursor containing `ut_annotated_object` as result
31+
* Range of data returned is limited by the input collection o cache object info
32+
*
33+
* @param a_cached_objects a `ut_annotation_objs_cache_info` list with information about objects to get from cache
34+
*/
35+
function get_annotations_for_objects(a_cached_objects ut_annotation_objs_cache_info) return sys_refcursor;
2236

23-
procedure cleanup_cache(a_objects ut_annotation_cached_objects);
37+
/**
38+
* Removes cached information about annotations for objects on the list and updates parse_time in cache info table.
39+
*
40+
* @param a_objects a `ut_annotation_objs_cache_info` list with information about objects to remove from cache
41+
*/
42+
procedure cleanup_cache(a_objects ut_annotation_objs_cache_info);
43+
44+
/**
45+
* Removes cached information about annotations for objects of specified type and specified owner
46+
*
47+
* @param a_object_owner owner of objects to purge annotations for
48+
* @param a_object_type type of objects to purge annotations for
49+
*/
50+
procedure purge_cache(a_object_owner varchar2, a_object_type varchar2);
2451

2552
end;
2653
/

‎source/core/annotations/ut_annotation_manager.pkb‎

Lines changed: 100 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -19,32 +19,30 @@ create or replace package body ut_annotation_manager as
1919
------------------------------
2020
--private definitions
2121

22-
function get_annotated_objects_cursor(a_object_owner varchar2, a_object_type varchar2, a_object_name varchar2) return sys_refcursor is
22+
function get_annotation_objs_info_cur(a_object_owner varchar2, a_object_type varchar2) return sys_refcursor is
2323
l_result sys_refcursor;
2424
l_ut_owner varchar2(250) := ut_utils.ut_owner;
2525
l_objects_view varchar2(200) := ut_metadata.get_dba_view('dba_objects');
2626
l_cursor_text long;
2727
begin
2828
l_cursor_text :=
29-
q'[select ]'||l_ut_owner||q'[.ut_annotation_cached_object(
29+
q'[select ]'||l_ut_owner||q'[.ut_annotation_obj_cache_info(
3030
object_owner => o.owner,
3131
object_name => o.object_name,
3232
object_type => o.object_type,
33-
needs_refresh => case when o.last_ddl_time < i.parse_time then 'N' else 'Y' end,
34-
cache_id => i.cache_id
33+
needs_refresh => case when o.last_ddl_time < i.parse_time then 'N' else 'Y' end
3534
)
3635
from ]'||l_objects_view||q'[ o
3736
left join ]'||l_ut_owner||q'[.ut_annotation_cache_info i
3837
on o.owner = i.object_owner and o.object_name = i.object_name and o.object_type = i.object_type
3938
where o.owner = :a_object_owner
4039
and o.object_type = :a_object_type
41-
and o.status = 'VALID'
42-
and :a_object_name ]'|| case when a_object_name is not null then '= o.object_name' else 'is null' end;
43-
open l_result for l_cursor_text using a_object_owner, a_object_type, a_object_name;
40+
and o.status = 'VALID' ]';
41+
open l_result for l_cursor_text using a_object_owner, a_object_type;
4442
return l_result;
4543
end;
4644

47-
function get_sources_for_annotations(a_object_owner varchar2, a_object_type varchar2, a_object_name varchar2) return sys_refcursor is
45+
function get_sources_to_annotate(a_object_owner varchar2, a_object_type varchar2) return sys_refcursor is
4846
l_result sys_refcursor;
4947
l_sources_view varchar2(200) := ut_metadata.get_dba_view('dba_source');
5048
begin
@@ -59,15 +57,14 @@ create or replace package body ut_annotation_manager as
5957
where x.type = :a_object_type
6058
and x.owner = :a_object_owner
6159
and x.text like '%--%\%%' escape '\'
62-
and :a_object_name ]'|| case when a_object_name is not null then '= x.name' else 'is null' end || q'[
6360
)
6461
order by name, line]'
65-
using a_object_type, a_object_owner, a_object_type, a_object_owner, a_object_name;
62+
using a_object_type, a_object_owner, a_object_type, a_object_owner;
6663

6764
return l_result;
6865
end;
6966

70-
function get_sources_for_annotations(a_object_owner varchar2, a_object_type varchar2, a_objects_to_refresh ut_annotation_cached_objects) return sys_refcursor is
67+
function get_sources_to_annotate(a_object_owner varchar2, a_object_type varchar2, a_objects_to_refresh ut_annotation_objs_cache_info) return sys_refcursor is
7168
l_result sys_refcursor;
7269
l_sources_view varchar2(200) := ut_metadata.get_dba_view('dba_source');
7370
l_card natural;
@@ -97,101 +94,117 @@ create or replace package body ut_annotation_manager as
9794
return l_result;
9895
end;
9996

100-
------------------------------------------------------------
101-
--public definitions
102-
------------------------------------------------------------
103-
function get_annotated_objects(a_object_owner varchar2, a_object_type varchar2, a_object_name varchar2 := null) return ut_annotated_objects pipelined is
104-
l_info_rows ut_annotation_cached_objects;
105-
l_in_cache ut_annotation_cached_objects;
106-
l_to_parse ut_annotation_cached_objects := ut_annotation_cached_objects();
107-
l_cursor sys_refcursor;
108-
l_results ut_annotated_objects;
109-
l_result ut_annotated_object;
110-
c_object_fetch_limit constant integer := 10;
97+
procedure build_annot_cache_for_sources(
98+
a_object_owner varchar2, a_object_type varchar2, a_sources_cursor sys_refcursor,
99+
a_schema_objects ut_annotation_objs_cache_info
100+
) is
101+
l_annotations ut_annotations;
111102
c_lines_fetch_limit constant integer := 1000;
112103
l_lines dbms_preprocessor.source_lines_t;
113104
l_names dbms_preprocessor.source_lines_t;
114-
l_name varchar2(250) := '''';
105+
l_name varchar2(250);
115106
l_object_lines dbms_preprocessor.source_lines_t;
107+
pragma autonomous_transaction;
116108
begin
117-
--get information about cached objects
118-
l_cursor := get_annotated_objects_cursor(a_object_owner, a_object_type, a_object_name);
119-
fetch l_cursor bulk collect into l_info_rows;
120-
close l_cursor;
121-
122-
--get list of objects in cache
123-
select value(x) bulk collect into l_in_cache from table(l_info_rows) x where x.needs_refresh = 'N';
124-
125-
--if not all in cache, get list of objects to refresh
126-
if l_in_cache.count <= l_info_rows.count then
127-
select value(x) bulk collect into l_to_parse from table(l_info_rows) x where x.needs_refresh = 'Y';
128-
end if;
109+
ut_annotation_cache_manager.cleanup_cache(a_schema_objects);
110+
loop
111+
fetch a_sources_cursor bulk collect into l_names, l_lines limit c_lines_fetch_limit;
112+
for i in 1 .. l_names.count loop
113+
if l_names(i) != l_name then
114+
l_annotations := ut_annotation_parser.parse_object_annotations(l_object_lines);
115+
ut_annotation_cache_manager.update_cache(
116+
ut_annotated_object(a_object_owner, l_name, a_object_type, l_annotations)
117+
);
118+
l_object_lines.delete;
119+
end if;
129120

130-
--pipe annotations from cache
131-
if l_in_cache.count > 0 then
132-
l_cursor := ut_annotation_cache_manager.get_annotations_for_objects(l_in_cache);
133-
loop
134-
fetch l_cursor bulk collect into l_results limit c_object_fetch_limit;
135-
for i in 1 .. l_results.count loop
136-
pipe row (l_results(i));
137-
end loop;
138-
exit when l_cursor%notfound;
121+
l_name := l_names(i);
122+
l_object_lines(l_object_lines.count+1) := l_lines(i);
139123
end loop;
140-
close l_cursor;
124+
exit when a_sources_cursor%notfound;
125+
126+
end loop;
127+
if a_sources_cursor%rowcount > 0 then
128+
l_annotations := ut_annotation_parser.parse_object_annotations(l_object_lines);
129+
ut_annotation_cache_manager.update_cache(
130+
ut_annotated_object(a_object_owner, l_name, a_object_type, l_annotations)
131+
);
132+
l_object_lines.delete;
141133
end if;
134+
close a_sources_cursor;
135+
commit;
136+
end;
142137

143-
--if some source needs parsing
144-
if l_to_parse.count > 0 then
145-
--do we need to parse all of sources
146-
if l_in_cache.count = 0 then
147-
l_cursor := get_sources_for_annotations(a_object_owner, a_object_type, a_object_name);
148-
else
149-
-- sources need to be filtered by objects to parse
150-
l_cursor := get_sources_for_annotations(a_object_owner, a_object_type, l_to_parse);
151-
end if;
152-
153-
--remove cached annotations data for objects that will be refreshed
154-
ut_annotation_cache_manager.cleanup_cache(l_to_parse);
155138

156-
loop
157-
fetch l_cursor bulk collect into l_names, l_lines limit c_lines_fetch_limit;
139+
procedure rebuild_annotation_cache( a_object_owner varchar2, a_object_type varchar2, a_info_rows ut_annotation_objs_cache_info) is
140+
l_objects_in_cache_count integer;
141+
l_objects_to_parse ut_annotation_objs_cache_info := ut_annotation_objs_cache_info();
142+
begin
143+
--get list of objects in cache
144+
select count( 1)into l_objects_in_cache_count from table(a_info_rows) x where x.needs_refresh = 'N';
158145

159-
for i in 1 .. l_names.count loop
146+
--if cache is empty and there are objects to parse
147+
if l_objects_in_cache_count = 0 and a_info_rows.count > 0 then
160148

161-
if l_names(i) != l_name then
162-
l_result := ut_annotated_object(
163-
a_object_owner, l_name, a_object_type,
164-
ut_annotation_parser.parse_object_annotations(l_object_lines)
165-
);
166-
ut_annotation_cache_manager.update_cache(l_result);
167-
if l_result.annotations.count > 0 then
168-
pipe row (l_result);
169-
end if;
170-
l_object_lines.delete;
171-
end if;
149+
build_annot_cache_for_sources(
150+
a_object_owner, a_object_type,
151+
--all schema objects
152+
get_sources_to_annotate(a_object_owner, a_object_type),
153+
a_info_rows
154+
);
172155

173-
l_name := l_names(i);
174-
l_object_lines(l_object_lines.count+1) := l_lines(i);
156+
--if not all in cache, get list of objects to rebuild cache for
157+
elsif l_objects_in_cache_count < a_info_rows.count then
175158

176-
end loop;
177-
exit when l_cursor%notfound;
159+
select value(x)bulk collect into l_objects_to_parse from table(a_info_rows) x where x.needs_refresh = 'Y';
178160

179-
end loop;
180-
181-
if l_name is not null then
182-
l_result := ut_annotated_object(
183-
a_object_owner, l_name, a_object_type,
184-
ut_annotation_parser.parse_object_annotations(l_object_lines)
161+
--if some source needs parsing and putting into cache
162+
if l_objects_to_parse.count > 0 then
163+
build_annot_cache_for_sources(
164+
a_object_owner, a_object_type,
165+
get_sources_to_annotate(a_object_owner, a_object_type, l_objects_to_parse),
166+
l_objects_to_parse
185167
);
186-
ut_annotation_cache_manager.update_cache(l_result);
187-
if l_result.annotations.count > 0 then
188-
pipe row (l_result);
189-
end if;
190-
l_object_lines.delete;
191168
end if;
192169

193-
close l_cursor;
194170
end if;
171+
end;
172+
173+
------------------------------------------------------------
174+
--public definitions
175+
------------------------------------------------------------
176+
procedure rebuild_annotation_cache(a_object_owner varchar2, a_object_type varchar2) is
177+
l_info_cursor sys_refcursor;
178+
l_info_rows ut_annotation_objs_cache_info;
179+
begin
180+
l_info_cursor := get_annotation_objs_info_cur(a_object_owner, a_object_type);
181+
fetch l_info_cursor bulk collect into l_info_rows;
182+
close l_info_cursor;
183+
rebuild_annotation_cache(a_object_owner, a_object_type, l_info_rows);
184+
end;
185+
186+
function get_annotated_objects(a_object_owner varchar2, a_object_type varchar2) return ut_annotated_objects pipelined is
187+
l_info_cursor sys_refcursor;
188+
l_info_rows ut_annotation_objs_cache_info;
189+
l_cursor sys_refcursor;
190+
l_results ut_annotated_objects;
191+
c_object_fetch_limit constant integer := 10;
192+
begin
193+
l_info_cursor := get_annotation_objs_info_cur(a_object_owner, a_object_type);
194+
fetch l_info_cursor bulk collect into l_info_rows;
195+
close l_info_cursor;
196+
rebuild_annotation_cache(a_object_owner, a_object_type, l_info_rows);
197+
198+
--pipe annotations from cache
199+
l_cursor := ut_annotation_cache_manager.get_annotations_for_objects(l_info_rows);
200+
loop
201+
fetch l_cursor bulk collect into l_results limit c_object_fetch_limit;
202+
for i in 1 .. l_results.count loop
203+
pipe row (l_results(i));
204+
end loop;
205+
exit when l_cursor%notfound;
206+
end loop;
207+
close l_cursor;
195208

196209
end;
197210

‎source/core/annotations/ut_annotation_manager.pks‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,26 @@ create or replace package ut_annotation_manager authid current_user as
2121
*/
2222

2323
/**
24-
* Parses an object or all objects of a specified type for database schema.
25-
* Results are returned in as a pipelined function.
26-
* @param a_object_owner schema name to be parsed
27-
* @param a_object_type type of object to be parsed
28-
* @param a_object_name name of object to be parsed - optional
24+
* Gets annotations for all objects of a specified type for database schema.
25+
* Annotations that are stale or missing are parsed and placed in persistent cache.
26+
* After placing in cache, annotation data is returned as pipelined table data.
27+
*
28+
* @param a_object_owner owner of objects to get annotations for
29+
* @param a_object_type type of objects to get annotations for
2930
* @return array containing annotated objects along with annotations for each object (nested)
3031
*/
31-
function get_annotated_objects(a_object_owner varchar2, a_object_type varchar2, a_object_name varchar2 := null) return ut_annotated_objects pipelined;
32+
function get_annotated_objects(a_object_owner varchar2, a_object_type varchar2) return ut_annotated_objects pipelined;
33+
34+
/**
35+
* Rebuilds annotation cache for a specified schema and object type.
36+
* The procedure is called internally by `get_annotated_objects` function.
37+
* It can be used to speedup initial execution of utPLSQL on a given schema
38+
* if it is executed before any call is made to `ut.run` or `ut_runner.run` procedure.
39+
*
40+
* @param a_object_owner owner of objects to get annotations for
41+
* @param a_object_type type of objects to get annotations for
42+
*/
43+
procedure rebuild_annotation_cache(a_object_owner varchar2, a_object_type varchar2);
3244

3345
end ut_annotation_manager;
3446
/

source/core/annotations/ut_annotation_cached_object.tps renamed to source/core/annotations/ut_annotation_obj_cache_info.tps

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
create type ut_annotation_cached_object as object(
1+
create type ut_annotation_obj_cache_info as object(
22
/*
33
utPLSQL - Version X.X.X.X
44
Copyright 2016 - 2017 utPLSQL Project
@@ -18,7 +18,6 @@ create type ut_annotation_cached_object as object(
1818
object_owner varchar2(250),
1919
object_name varchar2(250),
2020
object_type varchar2(250),
21-
needs_refresh varchar2(1),
22-
cache_id number(20,0)
21+
needs_refresh varchar2(1)
2322
)
2423
/

0 commit comments

Comments
 (0)