Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit f56cc49

Browse files
jgebaljgebal
authored
Merge pull request #901 from utPLSQL/feature/cache_improvements
Adding DDL trigger to speedup framework start
2 parents df67dee + 62d683a commit f56cc49

32 files changed

Lines changed: 816 additions & 254 deletions

‎.travis/install.sh‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ set verify off
1010
1111
--alter session set plsql_warnings = 'ENABLE:ALL', 'DISABLE:(5004,5018,6000,6001,6003,6009,6010,7206)';
1212
alter session set plsql_optimize_level=0;
13-
@install_headless.sql $UT3_OWNER $UT3_OWNER_PASSWORD
13+
@install_headless_with_trigger.sql $UT3_OWNER $UT3_OWNER_PASSWORD
1414
SQL
1515

1616
#Run this step only on second child job (12.1 - at it's fastest)
@@ -25,14 +25,18 @@ if [[ "${TRAVIS_JOB_NUMBER}" =~ \.2$ ]]; then
2525
set verify off
2626
2727
@uninstall_all.sql $UT3_OWNER
28+
whenever sqlerror exit failure rollback
2829
declare
2930
v_leftover_objects_count integer;
3031
begin
3132
select sum(cnt)
3233
into v_leftover_objects_count
33-
from (select count(1) cnt from dba_objects where owner = '$UT3_OWNER'
34-
union all
35-
select count(1) cnt from dba_synonyms where table_owner = '$UT3_OWNER'
34+
from (
35+
select count(1) cnt from dba_objects where owner = '$UT3_OWNER'
36+
where object_name not like 'PLSQL_PROFILER%' and object_name not like 'DBMSPCC_%'
37+
union all
38+
select count(1) cnt from dba_synonyms where table_owner = '$UT3_OWNER'
39+
where table_name not like 'PLSQL_PROFILER%' and table_name not like 'DBMSPCC_%'
3640
);
3741
if v_leftover_objects_count > 0 then
3842
raise_application_error(-20000, 'Not all objects were successfully uninstalled - leftover objects count='||v_leftover_objects_count);
@@ -47,6 +51,7 @@ SQL
4751
4852
alter session set plsql_optimize_level=0;
4953
@install.sql $UT3_OWNER
54+
@install_ddl_trigger.sql $UT3_OWNER
5055
@create_synonyms_and_grants_for_public.sql $UT3_OWNER
5156
SQL
5257

@@ -65,6 +70,9 @@ SQL
6570
time "$SQLCLI" sys/$ORACLE_PWD@//$CONNECTION_STR AS SYSDBA <<-SQL
6671
set feedback off
6772
@create_utplsql_owner.sql $UT3_TESTER $UT3_TESTER_PASSWORD $UT3_TABLESPACE
73+
--needed for disabling DDL trigger and testint parser without trigger enabled/present
74+
grant alter any trigger to ut3_tester;
75+
grant administer database trigger to $UT3_TESTER;
6876
exit
6977
SQL
7078

@@ -88,5 +96,7 @@ set feedback on
8896
--Needed for testing coverage outside of main UT3 schema.
8997
grant create any procedure, drop any procedure, execute any procedure, create any type, drop any type, execute any type, under any type, select any table, update any table, insert any table, delete any table, create any table, drop any table, alter any table, select any dictionary, create any synonym, drop any synonym to $UT3_TESTER_HELPER;
9098
grant create job to $UT3_TESTER_HELPER;
99+
--Needed to allow for enable/disable of annotation triggers
100+
grant administer database trigger to $UT3_TESTER_HELPER;
91101
exit
92102
SQL

‎.travis/install_utplsql_release.sh‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,14 @@ end;
3636
/
3737
SQL
3838

39+
INSTALL_FILE="install_headless_with_trigger.sql"
40+
if [[ ! -f "${INSTALL_FILE}" ]]; then
41+
INSTALL_FILE="install_headless.sql"
42+
fi
43+
3944
"$SQLCLI" sys/$ORACLE_PWD@//$CONNECTION_STR AS SYSDBA <<SQL
4045
alter session set plsql_optimize_level=0;
41-
@install_headless.sql ${UT3_RELEASE_VERSION_SCHEMA}
46+
@${INSTALL_FILE} ${UT3_RELEASE_VERSION_SCHEMA}
4247
exit
4348
SQL
4449

‎development/utplsql_style_check.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,8 @@ select i.name, i.type, i.object_name, i.object_type, i.usage, i.line, i.col, cou
4040
and i.usage_context_id = p.usage_id
4141
where i.type like 'VARIABLE' and i.usage = 'DECLARATION'
4242
and i.object_type not in ('TYPE')
43-
and (i.name not like 'L#_%' escape '#' and p.type in ('PROCEDURE','FUNCTION','ITERATOR')
44-
or i.name not like 'G#_%' escape '#' and p.type not in ('PROCEDURE','FUNCTION','ITERATOR'))
43+
and (i.name not like 'L#_%' escape '#' and p.type in ('PROCEDURE','FUNCTION','ITERATOR','TRIGGER')
44+
or i.name not like 'G#_%' escape '#' and p.type not in ('PROCEDURE','FUNCTION','ITERATOR','TRIGGER'))
4545
and p.type != 'RECORD'
4646
order by object_name, object_type, line, col
4747
;

‎docs/userguide/advanced_data_comparison.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -430,7 +430,7 @@ Unable to join sets:
430430
Please make sure that your join clause is not refferring to collection element
431431
```
432432

433-
***Note***
433+
**Note**
434434
>`join_by` option is slower to process as it needs to perform a cursor join.
435435
436436
## Defining item lists in option

‎docs/userguide/install.md‎

Lines changed: 75 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -72,16 +72,44 @@ The utPLSQL may be installed on any supported version of Oracle Database [see](h
7272
* 12c
7373
* 12c R2
7474
* 18c
75+
* 19c
7576

7677
# Headless installation
7778

78-
To install the utPLSQL into a new database schema and grant it to public, execute the script `install_headless.sql` as SYSDBA.
79+
utPLSQL can be installed with DDL trigger, to enable tracking of DDL changes to your unit test packages.
80+
This is the recommended installation approach, when you want to compile and run unit test packages in a schema containing huge amount of database packages (for example Oracle EBS installation schema).
81+
The reason for having DDL trigger is to enable in-time annotation parsing for utPLSQL.
82+
Without DDL trigger, utPLSQL needs to investigate your schema objects last_ddl_timestamp each time tests are executed to check if any of DB packages were changed in given schema and if they need scanning for annotation changes.
83+
This process can be time-consuming if DB schema is large.
7984

80-
The script accepts three optional parameters that define:
85+
The headless scripts accept three optional parameters that define:
8186
- username to create as owner of utPLSQL (default `ut3`)
8287
- password for owner of utPLSQL (default `XNtxj8eEgA6X6b6f`)
8388
- tablespace to use for storage of profiler data (default `users`)
8489

90+
The scripts need to be executed by `SYSDBA`, in order to grant access to `DBMS_LOCK` and `DBMS_CRYPTO` system packages.
91+
92+
**Note:**
93+
> Grant on `DBMS_LOCK` is required only for installation on Oracle versions below 18c. For versions 18c and above, utPLSQL uses `DBMS_SESSION.SLEEP` so access to `DBMS_LOCK` package is no longer needed.
94+
95+
**Note:**
96+
> The user performing the installation must have the `ADMINISTER DATABASE TRIGGER` privilege. This is required for installation of trigger that is responsible for parsing annotations at at compile-time of a package.
97+
98+
**Note:**
99+
> When installing with DDL trigger, utPLSQL will not be registering unit tests for any of oracle-maintained schemas.
100+
For Oracle 11g following users are excluded:
101+
> ANONYMOUS, APPQOSSYS, AUDSYS, DBSFWUSER, DBSNMP, DIP, GGSYS, GSMADMIN_INTERNAL, GSMCATUSER, GSMUSER, ORACLE_OCM, OUTLN, REMOTE_SCHEDULER_AGENT, SYS, SYS$UMF, SYSBACKUP, SYSDG, SYSKM, SYSRAC, SYSTEM, WMSYS, XDB, XS$NULL
102+
>
103+
> For Oracle 12c and above the users returned by below query are excluded by utPLSQL:
104+
>
105+
>```sql
106+
> select username from all_users where oracle_maintained='Y';
107+
>```
108+
109+
## Installation without DDL trigger
110+
111+
To install the utPLSQL into a new database schema and grant it to public, execute the script `install_headless.sql` as SYSDBA.
112+
85113
Example invocation of the script from command line:
86114
```bash
87115
cd source
@@ -94,16 +122,34 @@ cd source
94122
sqlplus sys/sys_pass@db as sysdba @install_headless.sql utp3 my_verySecret_password utp3_tablespace
95123
```
96124

97-
The script needs to be executed by `SYSDBA`, in order to grant access to `DBMS_LOCK` and `DBMS_CRYPTO` system packages.
125+
## Installation with DDL trigger
126+
127+
To install the utPLSQL into a new database schema and grant it to public, execute the script `install_headless_with_trigger.sql` as SYSDBA.
128+
129+
Example invocation of the script from command line:
130+
```bash
131+
cd source
132+
sqlplus sys/sys_pass@db as sysdba @install_headless_with_trigger.sql
133+
```
98134

99-
*Note:* Grant on `DBMS_LOCK` is required on Oracle versions below 18c
135+
Invoking script with parameters:
136+
```bash
137+
cd source
138+
sqlplus sys/sys_pass@db as sysdba @install_headless_with_trigger.sql utp3 my_verySecret_password utp3_tablespace
139+
```
100140

141+
**Note:**
142+
>When installing utPLSQL into database with existing unit test packages, utPLSQL will not be able to already-existing unit test packages. When utPSLQL was installed with DDL trigger, you have to do one of:
143+
>- Recompile existing Unit Test packages to make utPLSQL aware of their existence
144+
>- Invoke `exec ut_runner.rebuild_annotation_cache(a_schema_name=> ... );` for every schema containing unit tests in your database
145+
>
146+
> Steps above are required to assure annotation cache is populated properly from existing objects. Rebuilding annotation cache might be faster than code recompilation.
101147
102148
# Recommended Schema
103149
It is highly recommended to install utPLSQL in it's own schema. You are free to choose any name for this schema.
104150
Installing uPLSQL into shared schema is really not recommended as you loose isolation of framework.
105151

106-
If the installation and utPLSQL owner user is one and the same, the user must have the following Oracle system permissions before you can proceed with the installation.
152+
If the installing user and utPLSQL owner is one and the same, the user must have the following Oracle system permissions before you can proceed with the installation.
107153

108154
- CREATE SESSION
109155
- CREATE PROCEDURE
@@ -113,6 +159,7 @@ If the installation and utPLSQL owner user is one and the same, the user must ha
113159
- CREATE VIEW
114160
- CREATE SYNONYM
115161
- ALTER SESSION
162+
- CREATE TRIGGER
116163

117164
In addition the user must be granted the execute privilege on `DBMS_LOCK` and `DBMS_CRYPTO` packages.
118165

@@ -123,7 +170,7 @@ It is up to DBA to maintain the storage of the profiler tables.
123170

124171
# Manual installation procedure
125172

126-
### Creating schema for utPLSQL
173+
## Creating schema for utPLSQL
127174
To create the utPLSQL schema and grant all the required privileges execute script `create_utplsql_owner.sql` from the `source` directory with parameters:
128175

129176
- `user name` - the name of the user that will own of utPLSQL object
@@ -136,8 +183,8 @@ cd source
136183
sqlplus sys/sys_password@database as sysdba @create_utPLSQL_owner.sql ut3 ut3 users
137184
```
138185

139-
### Installing utPLSQL
140-
To install the utPLSQL framework into your database run the `/source/install.sql` script and provide `schema_name` where utPLSQL is to be installed.
186+
## Installing utPLSQL
187+
To install the utPLSQL framework into your database, go to `source` directory, run the `install.sql` providing the `schema_name` for utPLSQL as parameter.
141188
Schema must be created prior to calling the `install` script.
142189
You may install utPLSQL from any account that has sufficient privileges to create objects in other users schema.
143190

@@ -147,7 +194,26 @@ cd source
147194
sqlplus admin/admins_password@database @install.sql ut3
148195
```
149196

150-
### Allowing other users to access the utPLSQL framework
197+
## Installing DDL trigger
198+
To minimize startup time of utPLSQL framework (especially on a database with large schema) it is recommended to install utPLSQL DDL trigger to enable utPLSQL annotation to be updated at compile-time.
199+
200+
It's recommended to install DDL trigger when connected as `SYSDBA` user. Trigger is created in utPLSQL schema.
201+
If using the owner schema of utPLSQL to install trigger, the owner needs to have `ADMINISTER DATABASE TRIGGER` and `CREATE TRIGGER` system privileges.
202+
If using different user to install trigger, the user needs to have `ADMINISTER DATABASE TRIGGER` and `CREATE ANY TRIGGER` system privileges.
203+
204+
To install DDL trigger go to `source` directory, run the `install_ddl_trigger.sql` providing the `schema_name` for utPLSQL as parameter.
205+
206+
Example invocation:
207+
```bash
208+
cd source
209+
sqlplus admin/admins_password@database @install_ddl_trigger.sql ut3
210+
```
211+
212+
**Note:**
213+
>Trigger can be installed ant any point in time.
214+
215+
216+
## Allowing other users to access the utPLSQL framework
151217
In order to allow other users to access utPLSQL, synonyms must be created and privileges granted.
152218
You have two options:
153219

‎source/check_sys_grants.sql‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
1+
define expected_grants = "&1"
12
declare
2-
c_expected_grants constant dbmsoutput_linesarray
3-
:= dbmsoutput_linesarray(
4-
'CREATE TYPE','CREATE VIEW','CREATE SYNONYM','CREATE SEQUENCE','CREATE PROCEDURE','CREATE TABLE'
5-
);
3+
c_expected_grants constant dbmsoutput_linesarray := dbmsoutput_linesarray( &expected_grants );
64

75
l_expected_grants dbmsoutput_linesarray := c_expected_grants;
86
l_missing_grants varchar2(4000);
97
begin
108
if user != SYS_CONTEXT('userenv','current_schema') then
119
for i in 1 .. l_expected_grants.count loop
12-
l_expected_grants(i) := replace(l_expected_grants(i),' ',' ANY ');
10+
if l_expected_grants(i) != 'ADMINISTER DATABASE TRIGGER' then
11+
l_expected_grants(i) := replace(l_expected_grants(i),' ',' ANY ');
12+
end if;
1313
end loop;
1414
end if;
1515
select listagg(' - '||privilege,CHR(10)) within group(order by privilege)

‎source/core/annotations/ut_annotation_cache_info.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,6 @@ create table ut_annotation_cache_info (
1818
object_type varchar2(250) not null,
1919
parse_time timestamp not null,
2020
constraint ut_annotation_cache_info_pk primary key(cache_id),
21-
constraint ut_annotation_cache_info_uk unique (object_owner, object_name, object_type)
21+
constraint ut_annotation_cache_info_uk unique (object_owner, object_type, object_name)
2222
) organization index;
2323

‎source/core/annotations/ut_annotation_cache_manager.pkb‎

Lines changed: 56 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,40 @@ create or replace package body ut_annotation_cache_manager as
1818

1919
procedure update_cache(a_object ut_annotated_object) is
2020
l_cache_id integer;
21+
l_new_objects_count integer := 0;
2122
pragma autonomous_transaction;
2223
begin
23-
update ut_annotation_cache_info i
24-
set i.parse_time = systimestamp
25-
where (i.object_owner, i.object_name, i.object_type)
26-
in ((a_object.object_owner, a_object.object_name, a_object.object_type))
27-
returning cache_id into l_cache_id;
28-
if sql%rowcount = 0 then
29-
insert into ut_annotation_cache_info
30-
(cache_id, object_owner, object_name, object_type, parse_time)
31-
values (ut_annotation_cache_seq.nextval, a_object.object_owner, a_object.object_name, a_object.object_type, systimestamp)
24+
-- if not in trigger, or object has annotations
25+
if ora_sysevent is null or a_object.annotations is not null and a_object.annotations.count > 0 then
26+
27+
update ut_annotation_cache_info i
28+
set i.parse_time = systimestamp
29+
where (i.object_owner, i.object_name, i.object_type)
30+
in ((a_object.object_owner, a_object.object_name, a_object.object_type))
3231
returning cache_id into l_cache_id;
32+
33+
if sql%rowcount = 0 then
34+
35+
insert into ut_annotation_cache_info
36+
(cache_id, object_owner, object_name, object_type, parse_time)
37+
values (ut_annotation_cache_seq.nextval, a_object.object_owner, a_object.object_name, a_object.object_type, systimestamp)
38+
returning cache_id into l_cache_id;
39+
l_new_objects_count := 1;
40+
end if;
41+
3342
end if;
3443

35-
delete from ut_annotation_cache c
36-
where cache_id = l_cache_id;
44+
update ut_annotation_cache_schema s
45+
set s.object_count = s.object_count + l_new_objects_count, s.max_parse_time = systimestamp
46+
where s.object_type = a_object.object_type and s.object_owner = a_object.object_owner;
47+
48+
if sql%rowcount = 0 then
49+
insert into ut_annotation_cache_schema s
50+
(object_owner, object_type, object_count, max_parse_time)
51+
values (a_object.object_owner, a_object.object_type, l_new_objects_count, systimestamp);
52+
end if;
53+
54+
delete from ut_annotation_cache c where cache_id = l_cache_id;
3755

3856
if a_object.annotations is not null and a_object.annotations.count > 0 then
3957
insert into ut_annotation_cache
@@ -73,17 +91,32 @@ create or replace package body ut_annotation_cache_manager as
7391
commit;
7492
end;
7593

94+
function get_cache_schema_info(a_object_owner varchar2, a_object_type varchar2) return t_cache_schema_info is
95+
l_result t_cache_schema_info;
96+
begin
97+
begin
98+
select *
99+
into l_result
100+
from ut_annotation_cache_schema s
101+
where s.object_type = a_object_type and s.object_owner = a_object_owner;
102+
exception
103+
when no_data_found then
104+
null;
105+
end;
106+
return l_result;
107+
end;
108+
76109
procedure remove_from_cache(a_objects ut_annotation_objs_cache_info) is
77110
pragma autonomous_transaction;
78111
begin
79112

80113
delete from ut_annotation_cache_info i
81-
where exists (
82-
select 1 from table (a_objects) o
83-
where o.object_name = i.object_name
84-
and o.object_type = i.object_type
85-
and o.object_owner = i.object_owner
86-
);
114+
where exists (
115+
select 1 from table (a_objects) o
116+
where o.object_name = i.object_name
117+
and o.object_type = i.object_type
118+
and o.object_owner = i.object_owner
119+
);
87120

88121
commit;
89122
end;
@@ -139,6 +172,12 @@ create or replace package body ut_annotation_cache_manager as
139172
delete from ut_annotation_cache_info i
140173
where ' || l_filter
141174
using a_object_owner, a_object_type;
175+
176+
execute immediate '
177+
delete from ut_annotation_cache_schema s
178+
where ' || l_filter
179+
using a_object_owner, a_object_type;
180+
142181
commit;
143182
end;
144183

‎source/core/annotations/ut_annotation_cache_manager.pks‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ create or replace package ut_annotation_cache_manager authid definer as
1515
See the License for the specific language governing permissions and
1616
limitations under the License.
1717
*/
18-
18+
subtype t_cache_schema_info is ut_annotation_cache_schema%rowtype;
1919
/**
2020
* Populates cache with information about object and it's annotations
2121
* Cache information for individual object is modified by this code
@@ -34,6 +34,8 @@ create or replace package ut_annotation_cache_manager authid definer as
3434
*/
3535
function get_annotations_for_objects(a_cached_objects ut_annotation_objs_cache_info, a_parse_time timestamp) return sys_refcursor;
3636

37+
function get_cache_schema_info(a_object_owner varchar2, a_object_type varchar2) return t_cache_schema_info;
38+
3739
/**
3840
* Removes cached information about annotations for objects on the list and updates parse_time in cache info table.
3941
*

0 commit comments

Comments
 (0)