Added cache mechanism to ut_annotation_parser.

Changed the way schema is scanned, so that all annotations/sources are scanned with single execution of a query. TODO - change the way annotations are parsed, so that we can scan for "floating" annotations. Annotations that are on top of suite, but associated with procedure, are not valid suite-level annotations. This is why the package `old_tests/helpers/test_reporters` was updated.
utPLSQL · jgebal · Oct 26, 2017 · Oct 6, 2017 · Oct 6, 2017 · Oct 7, 2017
commit df2c76388a0b346c2a7f1fb4f495b5ffd51727e8
diff --git a/old_tests/helpers/test_reporters.pks b/old_tests/helpers/test_reporters.pks
@@ -2,6 +2,7 @@ create or replace package test_reporters
 as
   --%suite(A suite for testing different outcomes from reporters)
   --%suitepath(org.utplsql.utplsql.test)
+
   --%beforeall
   procedure beforeall;
 

diff --git a/old_tests/ut_suite_manager/ut_suite_manager.CacheInvalidaesOnPackageDrop.sql b/old_tests/ut_suite_manager/ut_suite_manager.CacheInvalidaesOnPackageDrop.sql
@@ -34,12 +34,12 @@ begin
     select * bulk collect into l_test_report from table(ut.run(user || '.tst_package_to_be_dropped'));
   exception
     when others then
-      if sqlerrm like '%tst_package_to_be_dropped%does not exist%' then
+      if sqlerrm like '%tst_package_to_be_dropped%not found%' then
         :test_result := ut_utils.tr_success;
       end if;
   end;
   if :test_result != ut_utils.tr_success or :test_result is null then
-    dbms_output.put_line('Failed: Expected exception with text like ''%tst_package_to_be_dropped%does not exist%'' but got:''' ||
+    dbms_output.put_line('Failed: Expected exception with text like ''%tst_package_to_be_dropped%not found%'' but got:''' ||
                          sqlerrm || '''');
   end if;
 end;

diff --git a/source/core/annotations/ut_annotated_object.tps b/source/core/annotations/ut_annotated_object.tps
@@ -0,0 +1,8 @@
+create type ut_annotated_object as object(
+  object_owner                  varchar2(250),
+  object_name                   varchar2(250),
+  object_type                   varchar2(50),
+  annotations                   ut_annotations
+)
+/
+
diff --git a/source/core/annotations/ut_annotated_objects.tps b/source/core/annotations/ut_annotated_objects.tps
@@ -0,0 +1,3 @@
+create type ut_annotated_objects as table of ut_annotated_object
+/
+
diff --git a/source/core/annotations/ut_annotation_cache.sql b/source/core/annotations/ut_annotation_cache.sql
@@ -0,0 +1,14 @@
+create table ut_annotation_cache (
+  cache_id                      number(20,0)   not null,
+  annotation_position           number(5,0)    not null,
+  annotation_name               varchar2(1000) not null,
+  annotation_text               varchar2(4000),
+  subobject_name                varchar2(250),
+--   subobject_name_not_null       varchar2(250) generated always as (nvl(subobject_name,'null')) not null,
+  constraint ut_annotation_cache_pk primary key(cache_id, annotation_position),
+--   constraint ut_annotation_cache_pk primary key(cache_id, annotation_position, subobject_name_not_null),
+  constraint ut_annotation_cache_fk foreign key(cache_id) references ut_annotation_cache_info(cache_id) on delete cascade
+);
+
+create index ut_annotation_cache_fk on ut_annotation_cache(cache_id);
+
diff --git a/source/core/annotations/ut_annotation_cache_info.sql b/source/core/annotations/ut_annotation_cache_info.sql
@@ -0,0 +1,12 @@
+create table ut_annotation_cache_info (
+  cache_id        number(20,0)  not null,
+  object_owner    varchar2(250) not null,
+  object_name     varchar2(250) not null,
+  object_type     varchar2(250) not null,
+  parse_time      date          not null,
+  is_annotated    varchar2(1)   not null,
+  constraint ut_annotation_cache_info_pk primary key(cache_id),
+  constraint ut_annotation_cache_info_uk unique (object_owner, object_name, object_type),
+  constraint ut_annotation_cache_info_ck1 check (is_annotated in ('Y','N'))
+) organization index;
+
diff --git a/source/core/annotations/ut_annotation_cache_manager.pkb b/source/core/annotations/ut_annotation_cache_manager.pkb
@@ -0,0 +1,47 @@
+create or replace package body ut_annotation_cache_manager as
+
+  procedure update_cache(a_object ut_annotated_object, a_cache_id integer) is
+    l_is_annotated   varchar2(1);
+    v_cache_id       integer := a_cache_id;
+    l_current_schema varchar2(250) := ut_utils.ut_owner;
+    pragma autonomous_transaction;
+  begin
+    if a_object.annotations is not null and a_object.annotations.count > 0 then
+      l_is_annotated := 'Y';
+    else
+      l_is_annotated := 'N';
+    end if;
+
+    if v_cache_id is not null then
+      update ut_annotation_cache_info i
+         set i.parse_time = sysdate,
+             i.is_annotated = l_is_annotated
+       where i.cache_id = v_cache_id;
+    else
+      insert into ut_annotation_cache_info
+             (cache_id, object_owner, object_name, object_type, parse_time, is_annotated)
+      values (ut_annotation_cache_seq.nextval, a_object.object_owner, a_object.object_name, a_object.object_type, sysdate, l_is_annotated)
+        returning cache_id into v_cache_id;
+    end if;
+
+    delete from ut_annotation_cache c
+      where cache_id = v_cache_id;
+
+    if l_is_annotated = 'Y' then
+--       begin
+      insert into ut_annotation_cache
+             (cache_id, annotation_position, annotation_name, annotation_text, subobject_name)
+      select v_cache_id, a.position, a.name, a.text, a.subobject_name
+        from table(a_object.annotations) a;
+      --TODO - duplicate annotations found?? - should not happen, getting standalone annotations need to happen after procedure annotations were parsed
+--       exception
+--         when others then
+--           dbms_output.put_line(xmltype(anydata.convertCollection(a_object.annotations)).getclobval);
+--           raise;
+--       end;
+    end if;
+    commit;
+  end;
+
+end;
+/
diff --git a/source/core/annotations/ut_annotation_cache_manager.pks b/source/core/annotations/ut_annotation_cache_manager.pks
@@ -0,0 +1,6 @@
+create or replace package ut_annotation_cache_manager authid definer as
+
+  procedure update_cache(a_object ut_annotated_object, a_cache_id integer);
+
+end;
+/
diff --git a/source/core/annotations/ut_annotation_cache_seq.sql b/source/core/annotations/ut_annotation_cache_seq.sql
@@ -0,0 +1,2 @@
+create sequence ut_annotation_cache_seq cache 20
+/
diff --git a/source/core/annotations/ut_annotation_parser.pkb b/source/core/annotations/ut_annotation_parser.pkb
@@ -230,15 +230,138 @@ create or replace package body ut_annotation_parser as
       from table(l_annotations) x
      order by x.position;
 
-    -- printing out parsed structure for debugging
-    $if $$ut_trace $then
-      print_parse_results(l_result);
-    $end
+--     -- printing out parsed structure for debugging
+--     $if $$ut_trace $then
+--       print_parse_results(l_result);
+--     $end
     return l_result;
   end parse_package_annotations;
 
-  ------------------------------
+  function get_post_processed_source(a_source_lines ut_varchar2_rows) return clob is
+    l_lines       sys.dbms_preprocessor.source_lines_t;
+    l_source      clob;
+  begin
+    --convert to preprocessor lines
+    for i in 1 .. a_source_lines.count loop
+      l_lines(i) := a_source_lines(i);
+    end loop;
+    --get post-processed source
+    l_lines := sys.dbms_preprocessor.get_post_processed_source(l_lines);
+    --convert to clob
+    for i in 1..l_lines.count loop
+      ut_utils.append_to_clob(l_source, replace(l_lines(i), chr(13)||chr(10), chr(10)));
+    end loop;
+    return l_source;
+  end;
+
+  ------------------------------------------------------------
   --public definitions
+  ------------------------------------------------------------
+
+  function parse_annotations(a_cursor t_object_sources_cur) return ut_annotated_objects pipelined is
+    l_rec         t_object_source;
+    l_source      clob;
+    l_annotations ut_annotations;
+    l_result      ut_annotated_object;
+    ex_package_is_wrapped exception;
+    pragma exception_init(ex_package_is_wrapped, -24241);
+
+  begin
+    if not a_cursor% isopen then
+      return;
+    end if;
+    loop
+
+      fetch a_cursor into l_rec;
+      exit when a_cursor%notfound;
+      begin
+        --convert to post-processed source clob
+        l_source := get_post_processed_source(l_rec.lines);
+        --parse annotations
+        l_annotations := parse_package_annotations(l_source);
+        dbms_lob.freetemporary(l_source);
+      exception
+        when ex_package_is_wrapped then
+        null;
+      end;
+      --convert to query results
+      l_result := ut_annotated_object( l_rec.owner, l_rec.name, l_rec.type, l_annotations);
+
+      ut_annotation_cache_manager.update_cache(l_result, l_rec.cache_id);
+      if l_annotations is not empty then
+        pipe row (l_result);
+      end if;
+    end loop;
+    close a_cursor;
+    return;
+  end;
+
+
+  function get_annotated_objects(a_object_owner varchar2, a_object_type varchar2) return ut_annotated_objects pipelined is
+    l_objects_view   varchar2(200) := ut_metadata.get_dba_view('dba_objects');
+    l_sources_view   varchar2(200) := ut_metadata.get_dba_view('dba_source');
+    l_obj            ut_annotated_object;
+    l_current_schema varchar2(250) := ut_utils.ut_owner;
+    l_cursor         sys_refcursor;
+    l_cursor_sql     varchar2(32767);
+  begin
+    l_cursor_sql :=
+      q'[with object_cache_info
+        as (select /*+ cardinality(i 10000) */ o.owner as object_owner, o.object_name, o.object_type, i.cache_id,
+                   case when o.last_ddl_time < i.parse_time then 'N' else 'Y' end as cache_stale
+              from ]'||l_objects_view||q'[ o
+              left join ]'||l_current_schema||q'[.ut_annotation_cache_info i
+                on o.owner = i.object_owner  and o.object_name = i.object_name and o.object_type = i.object_type
+             where o.object_type = :a_object_type and o.status = 'VALID' and o.owner = :a_object_owner
+           ),
+           obj_info_with_source
+        as (select /*+ cardinality(o 10000) */o.object_owner, o.object_name, o.object_type, o.cache_stale, o.cache_id, s.line, s.text
+              from object_cache_info o
+              left join ]'||l_sources_view||q'[ s
+                on s.name  = o.object_name
+               and s.type  = :a_object_type
+               and s.owner = :a_object_owner
+               and o.cache_stale = 'Y'
+           ),
+           obj_info_source_grouped
+        as (select o.object_owner, o.object_name, o.object_type, o.cache_stale, o.cache_id,
+                   cast(collect(o.text order by o.line) as ]'||l_current_schema||q'[.ut_varchar2_rows) as texts
+              from obj_info_with_source o
+             group by o.object_owner, o.object_type, o.object_name, o.cache_stale, cache_id
+             order by o.object_owner, o.object_type, o.object_name
+          )
+      select obj
+        from (
+          select ]'||l_current_schema||q'[.ut_annotated_object(o.object_owner, o.object_name, o.object_type,
+                 cast(collect(
+                       ]'||l_current_schema||q'[.ut_annotation(c.annotation_position, c.annotation_name, c.annotation_text, c.subobject_name) order by c.annotation_position )
+                      as ]'||l_current_schema||q'[.ut_annotations)
+                 ) as obj
+            from obj_info_source_grouped o
+            join ]'||l_current_schema||q'[.ut_annotation_cache c
+              on o.cache_id = c.cache_id
+           where o.cache_stale = 'N'
+           group by o.object_owner, o.object_name, o.object_type
+          union all
+          -- this query needs to be executed as last part of the union
+          -- as it is updating the cache_stale flag in an autonomous transaction
+          select value(c) as obj
+            from table(
+              ]'||l_current_schema||q'[.ut_annotation_parser.parse_annotations(
+                cursor(select o.object_owner, o.object_name, o.object_type, o.cache_id, o.texts from obj_info_source_grouped o where o.cache_stale = 'Y')
+              )
+            ) c
+        ) a
+      order by a.obj.object_owner, a.obj.object_type, a.obj.object_name]';
+    open l_cursor for l_cursor_sql using a_object_type, a_object_owner, a_object_type, a_object_owner;
+    loop
+      fetch l_cursor into l_obj;
+      exit when l_cursor%notfound;
+      pipe row (l_obj);
+    end loop;
+    close l_cursor;
+    return;
+  end;
 
   function get_package_annotations(a_owner_name varchar2, a_name varchar2) return ut_annotations is
     l_source clob;

diff --git a/source/core/annotations/ut_annotation_parser.pks b/source/core/annotations/ut_annotation_parser.pks
@@ -46,18 +46,34 @@ create or replace package ut_annotation_parser authid current_user as
   */
   type tt_annotation_params is table of typ_annotation_param index by pls_integer;
 
+  type t_object_source is record(
+    owner      varchar2(250),
+    name       varchar2(250),
+    type       varchar2(50),
+    cache_id   integer,
+    lines      ut_varchar2_rows
+  );
+
+  type t_object_sources_cur is ref cursor return t_object_source;
+
   /*
     INTERNAL USE ONLY
   */
   function parse_package_annotations(a_source clob) return ut_annotations;
 
+  function parse_annotations(a_cursor t_object_sources_cur) return ut_annotated_objects pipelined;
+
+  function get_annotated_objects(a_object_owner varchar2, a_object_type varchar2) return ut_annotated_objects pipelined;
+
   /*
     function: get_package_annotations
 
     get annotations for specified package specification and return its annotated schema
   */
   function get_package_annotations(a_owner_name varchar2, a_name varchar2) return ut_annotations;
 
+  function get_post_processed_source(a_source_lines ut_varchar2_rows) return clob;
+
 
   /*
     function: parse_annotation_params
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		create type ut_annotated_objects as table of ut_annotated_object
		/
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		create sequence ut_annotation_cache_seq cache 20
		/