Adding cache cleanup and "intelligent" join to all_source, only when …

…needed.
utPLSQL · jgebal · Nov 18, 2018 · Oct 30, 2018 · Nov 2, 2018 · Nov 4, 2018
commit 59f773835e44baf785079d837985d22eebb2013b
diff --git a/source/core/annotations/ut_annotation_cache_manager.pkb b/source/core/annotations/ut_annotation_cache_manager.pkb
@@ -74,6 +74,21 @@ create or replace package body ut_annotation_cache_manager as
     commit;
   end;
 
+  procedure remove_from_cache(a_objects ut_annotation_objs_cache_info) is
+    pragma autonomous_transaction;
+  begin
+
+    delete from ut_annotation_cache_info i
+      where exists (
+        select 1 from table (a_objects) o
+        where o.object_name = i.object_name
+        and o.object_type = i.object_type
+        and o.object_owner = i.object_owner
+      );
+
+    commit;
+  end;
+
   function get_annotations_for_objects(a_cached_objects ut_annotation_objs_cache_info, a_parse_time timestamp) return sys_refcursor is
     l_results     sys_refcursor;
   begin
@@ -92,7 +107,7 @@ create or replace package body ut_annotation_cache_manager as
         join ut_annotation_cache_info i
           on o.object_owner = i.object_owner and o.object_name = i.object_name and o.object_type = i.object_type
         join ut_annotation_cache c on i.cache_id = c.cache_id
-       where ]'|| case when a_parse_time is null then ':a_parse_date is null' else 'i.parse_time >= :a_parse_time' end ||q'[
+       where ]'|| case when a_parse_time is null then ':a_parse_date is null' else 'i.parse_time > :a_parse_time' end ||q'[
        group by i.object_owner, i.object_name, i.object_type, i.parse_time]'
     using a_cached_objects, a_parse_time;
     return l_results;

diff --git a/source/core/annotations/ut_annotation_cache_manager.pks b/source/core/annotations/ut_annotation_cache_manager.pks
@@ -37,10 +37,17 @@ create or replace package ut_annotation_cache_manager authid definer as
   /**
    * Removes cached information about annotations for objects on the list and updates parse_time in cache info table.
    *
-   * @param a_objects a `ut_annotation_objs_cache_info` list with information about objects to remove from cache
+   * @param a_objects a `ut_annotation_objs_cache_info` list with information about objects to remove annotations for
    */
   procedure cleanup_cache(a_objects ut_annotation_objs_cache_info);
 
+  /**
+   * Removes information about objects on the list
+   *
+   * @param a_objects a `ut_annotation_objs_cache_info` list with information about objects to remove from cache
+   */
+  procedure remove_from_cache(a_objects ut_annotation_objs_cache_info);
+
   /**
    * Removes cached information about annotations for objects of specified type and specified owner
    *

diff --git a/source/core/annotations/ut_annotation_manager.pkb b/source/core/annotations/ut_annotation_manager.pkb
@@ -19,6 +19,38 @@ create or replace package body ut_annotation_manager as
   ------------------------------
   --private definitions
 
+--   function get_missing_objects(a_object_owner varchar2, a_object_type varchar2) return ut_annotation_objs_cache_info is
+--     l_rows         sys_refcursor;
+--     l_ut_owner     varchar2(250) := ut_utils.ut_owner;
+--     l_objects_view varchar2(200) := ut_metadata.get_dba_view('dba_objects');
+--     l_cursor_text  varchar2(32767);
+--     l_result       ut_annotation_objs_cache_info;
+--   begin
+--     l_cursor_text :=
+--       q'[select ]'||l_ut_owner||q'[.ut_annotation_obj_cache_info(
+--                     object_owner => i.object_owner,
+--                     object_name => i.object_name,
+--                     object_type => i.object_type,
+--                     needs_refresh => null
+--                   )
+--            from ]'||l_ut_owner||q'[.ut_annotation_cache_info i
+--            where
+--              not exists (
+--                 select 1  from ]'||l_objects_view||q'[ o
+--                  where o.owner = i.object_owner
+--                    and o.object_name = i.object_name
+--                    and o.object_type = i.object_type
+--                    and o.owner = :a_object_owner
+--                    and o.object_type = :a_object_type
+--                 )
+--             and i.object_owner = :a_object_owner
+--             and i.object_type = :a_object_type]';
+--     open l_rows for l_cursor_text  using a_object_owner, a_object_type, a_object_owner, a_object_type;
+--     fetch l_rows bulk collect into l_result limit 1000000;
+--     close l_rows;
+--     return l_result;
+--   end;
+
   function get_annotation_objs_info(a_object_owner varchar2, a_object_type varchar2, a_parse_date timestamp := null) return ut_annotation_objs_cache_info is
     l_rows         sys_refcursor;
     l_ut_owner     varchar2(250) := ut_utils.ut_owner;
@@ -28,25 +60,36 @@ create or replace package body ut_annotation_manager as
   begin
     l_cursor_text :=
       q'[select ]'||l_ut_owner||q'[.ut_annotation_obj_cache_info(
-                    object_owner => o.owner,
-                    object_name => o.object_name,
-                    object_type => o.object_type,
-                    needs_refresh => case when o.last_ddl_time < cast(i.parse_time as date) then 'N' else 'Y' end
+                    object_owner => nvl( o.owner, i.object_owner ),
+                    object_name => nvl( o.object_name, i.object_name ),
+                    object_type => nvl( o.object_type, i.object_type ),
+                    needs_refresh =>
+                      case
+                        when o.last_ddl_time < cast(i.parse_time as date) then 'N'
+                        when o.owner is null then null
+                        else 'Y'
+                      end
                   )
-           from ]'||l_objects_view||q'[ o
-           left join ]'||l_ut_owner||q'[.ut_annotation_cache_info i
+           from (
+                 select * from ]'||l_objects_view||q'[ o
+                  where o.owner = :a_object_owner
+                    and o.object_type = :a_object_type
+                ) o
+           full outer join (
+                 select * from ]'||l_ut_owner||q'[.ut_annotation_cache_info i
+                  where i.object_owner = :a_object_owner
+                    and i.object_type = :a_object_type
+                ) i
              on o.owner = i.object_owner
             and o.object_name = i.object_name
             and o.object_type = i.object_type
-          where o.owner = :a_object_owner
-            and o.object_type = :a_object_type
-            and ]'
+          where ]'
       || case
            when a_parse_date is null
            then ':a_parse_date is null'
-           else 'o.last_ddl_time >= cast(:a_parse_date as date)'
+           else 'o.last_ddl_time >= cast(:a_parse_date as date) or o.last_ddl_time is null'
          end;
-    open l_rows for l_cursor_text  using a_object_owner, a_object_type, a_parse_date;
+    open l_rows for l_cursor_text  using a_object_owner, a_object_type, a_object_owner, a_object_type, a_parse_date;
     fetch l_rows bulk collect into l_result limit 1000000;
     close l_rows;
     return l_result;
@@ -106,8 +149,9 @@ create or replace package body ut_annotation_manager as
   end;
 
   procedure build_annot_cache_for_sources(
-    a_object_owner varchar2, a_object_type varchar2, a_sources_cursor sys_refcursor,
-    a_schema_objects ut_annotation_objs_cache_info
+    a_object_owner     varchar2,
+    a_object_type      varchar2,
+    a_sources_cursor   sys_refcursor
   ) is
     l_annotations         ut_annotations;
     c_lines_fetch_limit   constant integer := 1000;
@@ -118,7 +162,6 @@ create or replace package body ut_annotation_manager as
     l_parse_time          date := sysdate;
     pragma autonomous_transaction;
   begin
-    ut_annotation_cache_manager.cleanup_cache(a_schema_objects);
     loop
       fetch a_sources_cursor bulk collect into l_names, l_lines limit c_lines_fetch_limit;
       for i in 1 .. l_names.count loop
@@ -148,17 +191,37 @@ create or replace package body ut_annotation_manager as
   end;
 
 
-  procedure rebuild_annotation_cache( a_object_owner varchar2, a_object_type varchar2, a_info_rows ut_annotation_objs_cache_info) is
-    l_objects_to_parse       ut_annotation_objs_cache_info := ut_annotation_objs_cache_info();
+  procedure rebuild_annotation_cache(
+    a_object_owner varchar2,
+    a_object_type  varchar2,
+    a_info_rows    ut_annotation_objs_cache_info
+  ) is
+    l_objects_to_parse       ut_annotation_objs_cache_info;
+    l_objects_to_remove      ut_annotation_objs_cache_info;
   begin
-    select value(x)bulk collect into l_objects_to_parse from table(a_info_rows) x where x.needs_refresh = 'Y';
+    select value(x)bulk collect into l_objects_to_parse
+      from table(a_info_rows) x where x.needs_refresh = 'Y';
+
+    ut_annotation_cache_manager.cleanup_cache(l_objects_to_parse);
+
+    if sys_context('userenv','current_schema') = a_object_owner
+      or ut_metadata.is_object_visible('ut3.ut_utils')
+      or ut_metadata.is_object_visible('dba_objects')
+    then
+      select value(x)bulk collect into l_objects_to_remove
+        from table(a_info_rows) x where x.needs_refresh is null;
+
+      ut_annotation_cache_manager.remove_from_cache(l_objects_to_remove);
+--       ut_annotation_cache_manager.remove_from_cache(
+--         get_missing_objects(a_object_owner, a_object_type)
+--       );
+    end if;
 
     --if some source needs parsing and putting into cache
     if l_objects_to_parse.count > 0 then
       build_annot_cache_for_sources(
         a_object_owner, a_object_type,
-        get_sources_to_annotate(a_object_owner, a_object_type, l_objects_to_parse),
-        l_objects_to_parse
+        get_sources_to_annotate(a_object_owner, a_object_type, l_objects_to_parse)
       );
     end if;
   end;

diff --git a/source/core/ut_metadata.pkb b/source/core/ut_metadata.pkb
@@ -148,15 +148,34 @@ create or replace package body ut_metadata as
   end;
 
   function get_dba_view(a_dba_view_name varchar2) return varchar2 is
-    l_invalid_object_name exception;
     l_result              varchar2(128) := lower(a_dba_view_name);
+  begin
+    if not is_object_visible(a_dba_view_name) then
+      l_result := replace(l_result,'dba_','all_');
+    end if;
+     return l_result;
+  end;
+
+  function user_has_execute_any_proc return boolean is
+    l_ut_owner     varchar2(250) := ut_utils.ut_owner;
+    l_dummy        varchar2(250);
+  begin
+    execute immediate 'select '||l_ut_owner||'.ut_utils.ut_owner from dual'
+      into l_dummy;
+    return true;
+  exception
+    when others then
+      return false;
+  end;
+
+  function is_object_visible(a_object_name varchar2) return boolean is
+    l_invalid_object_name exception;
     pragma exception_init(l_invalid_object_name,-44002);
   begin
-    l_result := dbms_assert.sql_object_name(l_result);
-    return l_result;
+    return dbms_assert.sql_object_name(a_object_name) is not null;
   exception
     when l_invalid_object_name then
-      return replace(l_result,'dba_','all_');
+      return false;
   end;
 
   function package_exists_in_cur_schema(a_object_name varchar2) return boolean is

diff --git a/source/core/ut_metadata.pks b/source/core/ut_metadata.pks
@@ -69,6 +69,18 @@ create or replace package ut_metadata authid current_user as
    */
   function get_dba_view(a_dba_view_name varchar2) return varchar2;
 
+  /**
+   * Returns true if object is accessible to current user
+   * @param a_object_name fully qualified object name (with schema name)
+   */
+  function is_object_visible(a_object_name varchar2) return boolean;
+
+  /**
+   * Returns true if current user has execute any procedure privilege
+   * The check is performed by checking if user can execute ut_utils package
+   */
+  function user_has_execute_any_proc return boolean;
+
   /**
    * Returns true if given object is a package and it exists in current schema
    * @param a_object_name the name of the object to be checked

diff --git a/source/core/ut_suite_builder.pkb b/source/core/ut_suite_builder.pkb
@@ -1012,6 +1012,31 @@ create or replace package body ut_suite_builder is
     close a_suite_data_cursor;
   end;
 
+  function get_missing_objects(a_object_owner varchar2) return ut_varchar2_rows is
+    l_rows         sys_refcursor;
+    l_ut_owner     varchar2(250) := ut_utils.ut_owner;
+    l_objects_view varchar2(200) := ut_metadata.get_dba_view('dba_objects');
+    l_cursor_text  varchar2(32767);
+    l_result       ut_varchar2_rows;
+  begin
+    l_cursor_text :=
+      q'[select i.object_name
+           from ]'||l_ut_owner||q'[.ut_suite_cache_package i
+           where
+             not exists (
+                select 1  from ]'||l_objects_view||q'[ o
+                 where o.owner = i.object_owner
+                   and o.object_name = i.object_name
+                   and o.object_type = 'PACKAGE'
+                   and o.owner = :a_object_owner
+                )
+            and i.object_owner = :a_object_owner]';
+    open l_rows for l_cursor_text  using a_object_owner, a_object_owner;
+    fetch l_rows bulk collect into l_result limit 1000000;
+    close l_rows;
+    return l_result;
+  end;
+
   function get_cached_suite_data(
     a_object_owner   varchar2,
     a_path           varchar2 := null,
@@ -1177,8 +1202,9 @@ create or replace package body ut_suite_builder is
     a_object_name    varchar2 := null,
     a_procedure_name varchar2 := null
   ) return ut_suite_items is
-    l_annotations_cursor sys_refcursor;
-    l_suite_cache_time   timestamp;
+    l_annotations_cursor    sys_refcursor;
+    l_suite_cache_time      timestamp;
+    l_skip_all_objects_scan boolean := false;
   begin
     l_suite_cache_time := ut_suite_cache_manager.get_schema_parse_time(a_owner_name);
     open l_annotations_cursor for
@@ -1188,12 +1214,23 @@ create or replace package body ut_suite_builder is
       )x ]'
     using a_owner_name, l_suite_cache_time;
 
+    -- if current user is the onwer or current user has execute any procedure privilege
+    if sys_context('userenv','current_schema') = a_owner_name
+      or ut_metadata.is_object_visible('ut3.ut_utils')
+    then
+      l_skip_all_objects_scan := true;
+    end if;
+    if l_skip_all_objects_scan or ut_metadata.is_object_visible('dba_objects') then
+      ut_suite_cache_manager.remove_from_cache( a_owner_name, get_missing_objects(a_owner_name) );
+    end if;
+
     return build_suites_from_annotations(
       a_owner_name,
       l_annotations_cursor,
       a_path,
       a_object_name,
-      a_procedure_name
+      a_procedure_name,
+      l_skip_all_objects_scan
     );
   end;
 
@@ -1202,17 +1239,27 @@ create or replace package body ut_suite_builder is
     l_schema_names ut_varchar2_rows;
     l_object_names ut_varchar2_rows;
     l_ut_owner     varchar2(250) := ut_utils.ut_owner;
+    l_need_all_objects_scan boolean := true;
   begin
+    -- if current user is the onwer or current user has execute any procedure privilege
+    if ut_metadata.is_object_visible('ut3.ut_utils')
+      or (a_schema_names is not null and a_schema_names.count = 1
+         and sys_context('userenv','current_schema') = a_schema_names(1))
+    then
+      l_need_all_objects_scan := false;
+    end if;
+
     execute immediate 'select c.object_owner, c.object_name
         from '||l_ut_owner||q'[.ut_suite_cache_package c
              join table ( :a_schema_names ) s
-               on c.object_owner = upper(s.column_value)
+               on c.object_owner = upper(s.column_value)]'
+      || case when l_need_all_objects_scan then q'[
        where exists
             (select 1 from  all_objects a
               where a.owner = c.object_owner
                     and a.object_name = c.object_name
                     and a.object_type = 'PACKAGE')
-        ]'
+        ]' end
     bulk collect into l_schema_names, l_object_names using a_schema_names;
     l_results.extend( l_schema_names.count );
     for i in 1 .. l_schema_names.count loop

diff --git a/source/core/ut_suite_cache_manager.pkb b/source/core/ut_suite_cache_manager.pkb
@@ -135,5 +135,17 @@ create or replace package body ut_suite_cache_manager is
     commit;
   end;
 
+  procedure remove_from_cache(a_schema_name varchar2, a_objects ut_varchar2_rows) is
+    pragma autonomous_transaction;
+  begin
+    delete
+      from ut_suite_cache_package i
+     where i.object_owner = a_schema_name
+       and i.object_name in ( select column_value from table (a_objects) );
+
+    commit;
+  end;
+
+
 end ut_suite_cache_manager;
 /
diff --git a/source/core/ut_suite_cache_manager.pks b/source/core/ut_suite_cache_manager.pks
@@ -29,5 +29,7 @@ create or replace package ut_suite_cache_manager authid definer is
 
   function get_schema_parse_time(a_schema_name varchar2) return timestamp result_cache;
 
+  procedure remove_from_cache(a_schema_name varchar2, a_objects ut_varchar2_rows);
+
 end ut_suite_cache_manager;
 /