From 7bae9ff5f67d86ccd358e16086d08fc0f20f5463 Mon Sep 17 00:00:00 2001 From: Jacek Gebal Date: Sat, 30 Mar 2019 20:58:55 +0000 Subject: [PATCH 1/4] Fixed storage for clob in output bugger table - make it inline. Changed how delete from output buffer table is handled - use ROWID. Increased fetch size from 100 to 3000 rows for output buffer. --- .../output_buffers/ut_output_buffer_tmp.sql | 7 ++++--- .../output_buffers/ut_output_table_buffer.tpb | 19 ++++++++++--------- source/install.sql | 1 - 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/source/core/output_buffers/ut_output_buffer_tmp.sql b/source/core/output_buffers/ut_output_buffer_tmp.sql index e7cb49817..e452e769f 100644 --- a/source/core/output_buffers/ut_output_buffer_tmp.sql +++ b/source/core/output_buffers/ut_output_buffer_tmp.sql @@ -32,14 +32,15 @@ begin is_finished = 0 and (text is not null or item_type is not null ) or is_finished = 1 and text is null and item_type is null ), constraint ut_output_buffer_fk1 foreign key (output_id) references ut_output_buffer_info_tmp$(output_id) -) organization index overflow nologging initrans 100 '; +) nologging initrans 100 +'; begin execute immediate - v_table_sql || 'lob(text) store as securefile ut_output_text(retention none)'; + v_table_sql || 'lob(text) store as securefile ut_output_text(retention none enable storage in row)'; exception when e_non_assm then execute immediate - v_table_sql || 'lob(text) store as basicfile ut_output_text(pctversion 0)'; + v_table_sql || 'lob(text) store as basicfile ut_output_text(pctversion 0 enable storage in row)'; end; end; diff --git a/source/core/output_buffers/ut_output_table_buffer.tpb b/source/core/output_buffers/ut_output_table_buffer.tpb index 5fc71570c..e6c154267 100644 --- a/source/core/output_buffers/ut_output_table_buffer.tpb +++ b/source/core/output_buffers/ut_output_table_buffer.tpb @@ -78,8 +78,9 @@ create or replace type body ut_output_table_buffer is end; overriding member function get_lines(a_initial_timeout natural := null, a_timeout_sec natural := null) return ut_output_data_rows pipelined is + type t_rowid_tab is table of urowid; + l_message_rowids t_rowid_tab; l_buffer_data ut_output_data_rows; - l_message_ids ut_integer_list; l_finished_flags ut_integer_list; l_already_waited_for number(10,2) := 0; l_finished boolean := false; @@ -90,14 +91,14 @@ create or replace type body ut_output_table_buffer is lc_long_sleep_time constant number(1) := 1; --sleep for 1 s when waiting long lc_long_wait_time constant number(1) := 1; --waiting more than 1 sec l_sleep_time number(2,1) := lc_short_sleep_time; - lc_bulk_limit constant integer := 100; + lc_bulk_limit constant integer := 3000; - procedure remove_read_data(a_message_ids ut_integer_list) is + procedure remove_read_data(a_message_rowids t_rowid_tab) is pragma autonomous_transaction; begin - delete from ut_output_buffer_tmp a - where a.output_id = self.output_id - and a.message_id in (select column_value from table(a_message_ids)); + forall i in 1 .. a_message_rowids.count + delete from ut_output_buffer_tmp a + where rowid = a_message_rowids(i); commit; end; @@ -112,13 +113,13 @@ create or replace type body ut_output_table_buffer is begin while not l_finished loop with ordered_buffer as ( - select a.message_id, ut_output_data_row(a.text, a.item_type), is_finished + select a.rowid, ut_output_data_row(a.text, a.item_type), is_finished from ut_output_buffer_tmp a where a.output_id = self.output_id order by a.message_id ) select b.* - bulk collect into l_message_ids, l_buffer_data, l_finished_flags + bulk collect into l_message_rowids, l_buffer_data, l_finished_flags from ordered_buffer b where rownum <= lc_bulk_limit; @@ -147,7 +148,7 @@ create or replace type body ut_output_table_buffer is exit; end if; end loop; - remove_read_data(l_message_ids); + remove_read_data(l_message_rowids); end if; if l_finished or l_already_waited_for >= l_wait_for then remove_buffer_info(); diff --git a/source/install.sql b/source/install.sql index e34aa57af..c80eb67e6 100644 --- a/source/install.sql +++ b/source/install.sql @@ -194,7 +194,6 @@ prompt Installing DBMSPLSQL Tables objects into &&ut3_owner schema @@install_component.sql 'expectations/data_values/ut_cursor_column.tps' @@install_component.sql 'expectations/data_values/ut_cursor_column_tab.tps' @@install_component.sql 'expectations/data_values/ut_cursor_details.tps' -@@install_component.sql 'expectations/data_values/ut_data_value_anydata.tps' @@install_component.sql 'expectations/data_values/ut_data_value_blob.tps' @@install_component.sql 'expectations/data_values/ut_data_value_boolean.tps' @@install_component.sql 'expectations/data_values/ut_data_value_clob.tps' From 9d6bfe8a61a70597d278a002c01946d24cf88d18 Mon Sep 17 00:00:00 2001 From: Jacek Gebal Date: Sun, 31 Mar 2019 13:24:41 +0100 Subject: [PATCH 2/4] Improved performance for reading data from output buffer. Decreased read limit back to 1000 as it should not have significant performance impact anymore. --- .../core/output_buffers/ut_message_id_seq.sql | 1 - .../output_buffers/ut_output_buffer_base.tps | 8 ++--- .../output_buffers/ut_output_table_buffer.tpb | 32 +++++++++++-------- .../output_buffers/ut_output_table_buffer.tps | 9 +++--- source/install.sql | 1 - source/uninstall_objects.sql | 2 -- 6 files changed, 28 insertions(+), 25 deletions(-) delete mode 100644 source/core/output_buffers/ut_message_id_seq.sql diff --git a/source/core/output_buffers/ut_message_id_seq.sql b/source/core/output_buffers/ut_message_id_seq.sql deleted file mode 100644 index 3f0cd25c4..000000000 --- a/source/core/output_buffers/ut_message_id_seq.sql +++ /dev/null @@ -1 +0,0 @@ -create sequence ut_message_id_seq nocycle cache 100; diff --git a/source/core/output_buffers/ut_output_buffer_base.tps b/source/core/output_buffers/ut_output_buffer_base.tps index 42c4a0d72..59226a7fb 100644 --- a/source/core/output_buffers/ut_output_buffer_base.tps +++ b/source/core/output_buffers/ut_output_buffer_base.tps @@ -18,10 +18,10 @@ create or replace type ut_output_buffer_base authid definer as object( output_id raw(32), member procedure init(self in out nocopy ut_output_buffer_base), - not instantiable member procedure close(self in ut_output_buffer_base), - not instantiable member procedure send_line(self in ut_output_buffer_base, a_text varchar2, a_item_type varchar2 := null), - not instantiable member procedure send_lines(self in ut_output_buffer_base, a_text_list ut_varchar2_rows, a_item_type varchar2 := null), - not instantiable member procedure send_clob(self in ut_output_buffer_base, a_text clob, a_item_type varchar2 := null), + not instantiable member procedure close(self in out nocopy ut_output_buffer_base), + not instantiable member procedure send_line(self in out nocopy ut_output_buffer_base, a_text varchar2, a_item_type varchar2 := null), + not instantiable member procedure send_lines(self in out nocopy ut_output_buffer_base, a_text_list ut_varchar2_rows, a_item_type varchar2 := null), + not instantiable member procedure send_clob(self in out nocopy ut_output_buffer_base, a_text clob, a_item_type varchar2 := null), not instantiable member function get_lines(a_initial_timeout natural := null, a_timeout_sec natural := null) return ut_output_data_rows pipelined, not instantiable member function get_lines_cursor(a_initial_timeout natural := null, a_timeout_sec natural := null) return sys_refcursor, not instantiable member procedure lines_to_dbms_output(self in ut_output_buffer_base, a_initial_timeout natural := null, a_timeout_sec natural := null) diff --git a/source/core/output_buffers/ut_output_table_buffer.tpb b/source/core/output_buffers/ut_output_table_buffer.tpb index e6c154267..41a3aa82f 100644 --- a/source/core/output_buffers/ut_output_table_buffer.tpb +++ b/source/core/output_buffers/ut_output_table_buffer.tpb @@ -20,6 +20,7 @@ create or replace type body ut_output_table_buffer is begin self.output_id := coalesce(a_output_id, sys_guid()); self.start_date := sysdate; + self.last_message_id := 0; self.init(); self.cleanup_buffer(); return; @@ -38,41 +39,44 @@ create or replace type body ut_output_table_buffer is commit; end; - overriding member procedure close(self in ut_output_table_buffer) is + overriding member procedure close(self in out nocopy ut_output_table_buffer) is pragma autonomous_transaction; begin + self.last_message_id := self.last_message_id + 1; insert into ut_output_buffer_tmp(output_id, message_id, is_finished) - values (self.output_id, ut_message_id_seq.nextval, 1); + values (self.output_id, self.last_message_id, 1); commit; end; - overriding member procedure send_line(self in ut_output_table_buffer, a_text varchar2, a_item_type varchar2 := null) is + overriding member procedure send_line(self in out nocopy ut_output_table_buffer, a_text varchar2, a_item_type varchar2 := null) is pragma autonomous_transaction; begin if a_text is not null or a_item_type is not null then + self.last_message_id := self.last_message_id + 1; insert into ut_output_buffer_tmp(output_id, message_id, text, item_type) - values (self.output_id, ut_message_id_seq.nextval, a_text, a_item_type); + values (self.output_id, self.last_message_id, a_text, a_item_type); end if; commit; end; - overriding member procedure send_lines(self in ut_output_table_buffer, a_text_list ut_varchar2_rows, a_item_type varchar2 := null) is + overriding member procedure send_lines(self in out nocopy ut_output_table_buffer, a_text_list ut_varchar2_rows, a_item_type varchar2 := null) is pragma autonomous_transaction; begin insert into ut_output_buffer_tmp(output_id, message_id, text, item_type) - select self.output_id, ut_message_id_seq.nextval, t.column_value, a_item_type + select self.output_id, self.last_message_id + rownum, t.column_value, a_item_type from table(a_text_list) t where t.column_value is not null or a_item_type is not null; - + self.last_message_id := self.last_message_id + a_text_list.count; commit; end; - overriding member procedure send_clob(self in ut_output_table_buffer, a_text clob, a_item_type varchar2 := null) is + overriding member procedure send_clob(self in out nocopy ut_output_table_buffer, a_text clob, a_item_type varchar2 := null) is pragma autonomous_transaction; begin if a_text is not null and a_text != empty_clob() or a_item_type is not null then + self.last_message_id := self.last_message_id + 1; insert into ut_output_buffer_tmp(output_id, message_id, text, item_type) - values (self.output_id, ut_message_id_seq.nextval, a_text, a_item_type); + values (self.output_id, self.last_message_id, a_text, a_item_type); end if; commit; end; @@ -91,7 +95,8 @@ create or replace type body ut_output_table_buffer is lc_long_sleep_time constant number(1) := 1; --sleep for 1 s when waiting long lc_long_wait_time constant number(1) := 1; --waiting more than 1 sec l_sleep_time number(2,1) := lc_short_sleep_time; - lc_bulk_limit constant integer := 3000; + lc_bulk_limit constant integer := 1000; + l_max_message_id integer := lc_bulk_limit; procedure remove_read_data(a_message_rowids t_rowid_tab) is pragma autonomous_transaction; @@ -113,15 +118,15 @@ create or replace type body ut_output_table_buffer is begin while not l_finished loop with ordered_buffer as ( - select a.rowid, ut_output_data_row(a.text, a.item_type), is_finished + select /*+ index(a) */ a.rowid, ut_output_data_row(a.text, a.item_type), is_finished from ut_output_buffer_tmp a where a.output_id = self.output_id + and a.message_id <= l_max_message_id order by a.message_id ) select b.* bulk collect into l_message_rowids, l_buffer_data, l_finished_flags - from ordered_buffer b - where rownum <= lc_bulk_limit; + from ordered_buffer b; --nothing fetched from output, wait and try again if l_buffer_data.count = 0 then @@ -159,6 +164,7 @@ create or replace type body ut_output_table_buffer is ); end if; end if; + l_max_message_id := l_max_message_id + lc_bulk_limit; end loop; return; end; diff --git a/source/core/output_buffers/ut_output_table_buffer.tps b/source/core/output_buffers/ut_output_table_buffer.tps index 1c9acbd1d..9cf64927c 100644 --- a/source/core/output_buffers/ut_output_table_buffer.tps +++ b/source/core/output_buffers/ut_output_table_buffer.tps @@ -17,12 +17,13 @@ create or replace type ut_output_table_buffer under ut_output_buffer_base ( */ start_date date, + last_message_id number(38,0), constructor function ut_output_table_buffer(self in out nocopy ut_output_table_buffer, a_output_id raw := null) return self as result, overriding member procedure init(self in out nocopy ut_output_table_buffer), - overriding member procedure send_line(self in ut_output_table_buffer, a_text varchar2, a_item_type varchar2 := null), - overriding member procedure send_lines(self in ut_output_table_buffer, a_text_list ut_varchar2_rows, a_item_type varchar2 := null), - overriding member procedure send_clob(self in ut_output_table_buffer, a_text clob, a_item_type varchar2 := null), - overriding member procedure close(self in ut_output_table_buffer), + overriding member procedure send_line(self in out nocopy ut_output_table_buffer, a_text varchar2, a_item_type varchar2 := null), + overriding member procedure send_lines(self in out nocopy ut_output_table_buffer, a_text_list ut_varchar2_rows, a_item_type varchar2 := null), + overriding member procedure send_clob(self in out nocopy ut_output_table_buffer, a_text clob, a_item_type varchar2 := null), + overriding member procedure close(self in out nocopy ut_output_table_buffer), overriding member function get_lines(a_initial_timeout natural := null, a_timeout_sec natural := null) return ut_output_data_rows pipelined, overriding member function get_lines_cursor(a_initial_timeout natural := null, a_timeout_sec natural := null) return sys_refcursor, overriding member procedure lines_to_dbms_output(self in ut_output_table_buffer, a_initial_timeout natural := null, a_timeout_sec natural := null), diff --git a/source/install.sql b/source/install.sql index c80eb67e6..08a5b59d7 100644 --- a/source/install.sql +++ b/source/install.sql @@ -99,7 +99,6 @@ alter session set current_schema = &&ut3_owner; --output buffer table @@install_component.sql 'core/output_buffers/ut_output_buffer_info_tmp.sql' @@install_component.sql 'core/output_buffers/ut_output_buffer_tmp.sql' -@@install_component.sql 'core/output_buffers/ut_message_id_seq.sql' --output buffer table api @@install_component.sql 'core/output_buffers/ut_output_table_buffer.tps' @@install_component.sql 'core/output_buffers/ut_output_table_buffer.tpb' diff --git a/source/uninstall_objects.sql b/source/uninstall_objects.sql index 62d14cb58..ed84aa0a0 100644 --- a/source/uninstall_objects.sql +++ b/source/uninstall_objects.sql @@ -257,8 +257,6 @@ drop view ut_output_buffer_info_tmp; drop table ut_output_buffer_info_tmp$; -drop sequence ut_message_id_seq; - drop type ut_output_data_rows force; drop type ut_output_data_row force; From 4d00c33bbd0cab1ed68159fec24cec342dfa932b Mon Sep 17 00:00:00 2001 From: Jacek Gebal Date: Sun, 31 Mar 2019 14:31:54 +0100 Subject: [PATCH 3/4] Increased read limit to 5000. Fixed bug with increasing `max_message_id` even when nothing was read. --- source/core/output_buffers/ut_output_table_buffer.tpb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/core/output_buffers/ut_output_table_buffer.tpb b/source/core/output_buffers/ut_output_table_buffer.tpb index 41a3aa82f..5a9bdcce7 100644 --- a/source/core/output_buffers/ut_output_table_buffer.tpb +++ b/source/core/output_buffers/ut_output_table_buffer.tpb @@ -95,7 +95,7 @@ create or replace type body ut_output_table_buffer is lc_long_sleep_time constant number(1) := 1; --sleep for 1 s when waiting long lc_long_wait_time constant number(1) := 1; --waiting more than 1 sec l_sleep_time number(2,1) := lc_short_sleep_time; - lc_bulk_limit constant integer := 1000; + lc_bulk_limit constant integer := 5000; l_max_message_id integer := lc_bulk_limit; procedure remove_read_data(a_message_rowids t_rowid_tab) is @@ -154,6 +154,7 @@ create or replace type body ut_output_table_buffer is end if; end loop; remove_read_data(l_message_rowids); + l_max_message_id := l_max_message_id + lc_bulk_limit; end if; if l_finished or l_already_waited_for >= l_wait_for then remove_buffer_info(); @@ -164,7 +165,6 @@ create or replace type body ut_output_table_buffer is ); end if; end if; - l_max_message_id := l_max_message_id + lc_bulk_limit; end loop; return; end; From e1119c659ca09440414d96ca935c4a2a5f96ca12 Mon Sep 17 00:00:00 2001 From: Jacek Gebal Date: Sun, 31 Mar 2019 17:34:50 +0100 Subject: [PATCH 4/4] Split output buffers into two. Now we have separate buffer for CLOB data and VARCHAR2 text data. --- .../output_buffers/ut_output_buffer_tmp.sql | 24 +- .../ut_output_clob_buffer_tmp.sql | 89 ++++++++ .../ut_output_clob_table_buffer.tpb | 214 ++++++++++++++++++ .../ut_output_clob_table_buffer.tps | 32 +++ .../output_buffers/ut_output_table_buffer.tpb | 76 ++++--- .../create_synonyms_and_grants_for_public.sql | 2 + source/create_user_grants.sql | 1 + source/create_user_synonyms.sql | 1 + source/install.sql | 3 + source/reporters/ut_debug_reporter.tpb | 2 +- source/reporters/ut_realtime_reporter.tpb | 2 +- source/uninstall_objects.sql | 6 + .../core/reporters/test_realtime_reporter.pkb | 2 +- test/core/test_output_buffer.pkb | 2 +- 14 files changed, 412 insertions(+), 44 deletions(-) create mode 100644 source/core/output_buffers/ut_output_clob_buffer_tmp.sql create mode 100644 source/core/output_buffers/ut_output_clob_table_buffer.tpb create mode 100644 source/core/output_buffers/ut_output_clob_table_buffer.tps diff --git a/source/core/output_buffers/ut_output_buffer_tmp.sql b/source/core/output_buffers/ut_output_buffer_tmp.sql index e452e769f..d596a4908 100644 --- a/source/core/output_buffers/ut_output_buffer_tmp.sql +++ b/source/core/output_buffers/ut_output_buffer_tmp.sql @@ -24,7 +24,7 @@ begin */ output_id raw(32) not null, message_id number(38,0) not null, - text clob, + text varchar2(4000), item_type varchar2(1000), is_finished number(1,0) default 0 not null, constraint ut_output_buffer_tmp_pk primary key(output_id, message_id), @@ -32,17 +32,19 @@ begin is_finished = 0 and (text is not null or item_type is not null ) or is_finished = 1 and text is null and item_type is null ), constraint ut_output_buffer_fk1 foreign key (output_id) references ut_output_buffer_info_tmp$(output_id) -) nologging initrans 100 +) organization index nologging initrans 100 + overflow nologging initrans 100 '; - begin - execute immediate - v_table_sql || 'lob(text) store as securefile ut_output_text(retention none enable storage in row)'; - exception - when e_non_assm then - execute immediate - v_table_sql || 'lob(text) store as basicfile ut_output_text(pctversion 0 enable storage in row)'; - - end; + execute immediate v_table_sql; +-- begin +-- execute immediate +-- v_table_sql || 'lob(text) store as securefile ut_output_text(retention none enable storage in row)'; +-- exception +-- when e_non_assm then +-- execute immediate +-- v_table_sql || 'lob(text) store as basicfile ut_output_text(pctversion 0 enable storage in row)'; +-- +-- end; end; / diff --git a/source/core/output_buffers/ut_output_clob_buffer_tmp.sql b/source/core/output_buffers/ut_output_clob_buffer_tmp.sql new file mode 100644 index 000000000..57be1c6f4 --- /dev/null +++ b/source/core/output_buffers/ut_output_clob_buffer_tmp.sql @@ -0,0 +1,89 @@ +declare + v_table_sql varchar2(32767); + e_non_assm exception; + pragma exception_init(e_non_assm, -43853); +begin + v_table_sql := 'create table ut_output_clob_buffer_tmp$( + /* + utPLSQL - Version 3 + Copyright 2016 - 2018 utPLSQL Project + Licensed under the Apache License, Version 2.0 (the "License"): + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + /* + * This table is not a global temporary table as it needs to allow cross-session data exchange + * It is used however as a temporary table with multiple writers. + * This is why it has very high initrans and has nologging + */ + output_id raw(32) not null, + message_id number(38,0) not null, + text clob, + item_type varchar2(1000), + is_finished number(1,0) default 0 not null, + constraint ut_output_clob_buffer_tmp_pk primary key(output_id, message_id), + constraint ut_output_clob_buffer_tmp_ck check( + is_finished = 0 and (text is not null or item_type is not null ) + or is_finished = 1 and text is null and item_type is null ), + constraint ut_output_clob_buffer_tmp_fk1 foreign key (output_id) references ut_output_buffer_info_tmp$(output_id) +) nologging initrans 100 +'; + begin + execute immediate + v_table_sql || 'lob(text) store as securefile ut_output_text(retention none enable storage in row)'; + exception + when e_non_assm then + execute immediate + v_table_sql || 'lob(text) store as basicfile ut_output_text(pctversion 0 enable storage in row)'; + + end; +end; +/ + +-- This is needed to be EBR ready as editioning view can only be created by edition enabled user +declare + ex_nonedition_user exception; + ex_view_doesnt_exist exception; + pragma exception_init(ex_nonedition_user,-42314); + pragma exception_init(ex_view_doesnt_exist,-942); + v_view_source varchar2(32767); +begin + begin + execute immediate 'drop view ut_output_clob_buffer_tmp'; + exception + when ex_view_doesnt_exist then + null; + end; + v_view_source := ' ut_output_clob_buffer_tmp as +/* +utPLSQL - Version 3 +Copyright 2016 - 2018 utPLSQL Project +Licensed under the Apache License, Version 2.0 (the "License"): +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +select output_id + ,message_id + ,text + ,item_type + ,is_finished + from ut_output_clob_buffer_tmp$'; + + execute immediate 'create or replace editioning view '||v_view_source; +exception + when ex_nonedition_user then + execute immediate 'create or replace view '||v_view_source; +end; +/ diff --git a/source/core/output_buffers/ut_output_clob_table_buffer.tpb b/source/core/output_buffers/ut_output_clob_table_buffer.tpb new file mode 100644 index 000000000..7b97c70ca --- /dev/null +++ b/source/core/output_buffers/ut_output_clob_table_buffer.tpb @@ -0,0 +1,214 @@ +create or replace type body ut_output_clob_table_buffer is + /* + utPLSQL - Version 3 + Copyright 2016 - 2018 utPLSQL Project + + Licensed under the Apache License, Version 2.0 (the "License"): + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + + constructor function ut_output_clob_table_buffer(self in out nocopy ut_output_clob_table_buffer, a_output_id raw := null) return self as result is + begin + self.output_id := coalesce(a_output_id, sys_guid()); + self.start_date := sysdate; + self.last_message_id := 0; + self.init(); + self.cleanup_buffer(); + return; + end; + + overriding member procedure init(self in out nocopy ut_output_clob_table_buffer) is + pragma autonomous_transaction; + l_exists int; + begin + select count(*) into l_exists from ut_output_buffer_info_tmp where output_id = self.output_id; + if ( l_exists > 0 ) then + update ut_output_buffer_info_tmp set start_date = self.start_date where output_id = self.output_id; + else + insert into ut_output_buffer_info_tmp(output_id, start_date) values (self.output_id, self.start_date); + end if; + commit; + end; + + overriding member procedure close(self in out nocopy ut_output_clob_table_buffer) is + pragma autonomous_transaction; + begin + self.last_message_id := self.last_message_id + 1; + insert into ut_output_clob_buffer_tmp(output_id, message_id, is_finished) + values (self.output_id, self.last_message_id, 1); + commit; + end; + + overriding member procedure send_line(self in out nocopy ut_output_clob_table_buffer, a_text varchar2, a_item_type varchar2 := null) is + pragma autonomous_transaction; + begin + if a_text is not null or a_item_type is not null then + self.last_message_id := self.last_message_id + 1; + insert into ut_output_clob_buffer_tmp(output_id, message_id, text, item_type) + values (self.output_id, self.last_message_id, a_text, a_item_type); + end if; + commit; + end; + + overriding member procedure send_lines(self in out nocopy ut_output_clob_table_buffer, a_text_list ut_varchar2_rows, a_item_type varchar2 := null) is + pragma autonomous_transaction; + begin + insert into ut_output_clob_buffer_tmp(output_id, message_id, text, item_type) + select self.output_id, self.last_message_id + rownum, t.column_value, a_item_type + from table(a_text_list) t + where t.column_value is not null or a_item_type is not null; + self.last_message_id := self.last_message_id + a_text_list.count; + commit; + end; + + overriding member procedure send_clob(self in out nocopy ut_output_clob_table_buffer, a_text clob, a_item_type varchar2 := null) is + pragma autonomous_transaction; + begin + if a_text is not null and a_text != empty_clob() or a_item_type is not null then + self.last_message_id := self.last_message_id + 1; + insert into ut_output_clob_buffer_tmp(output_id, message_id, text, item_type) + values (self.output_id, self.last_message_id, a_text, a_item_type); + end if; + commit; + end; + + overriding member function get_lines(a_initial_timeout natural := null, a_timeout_sec natural := null) return ut_output_data_rows pipelined is + type t_rowid_tab is table of urowid; + l_message_rowids t_rowid_tab; + l_buffer_data ut_output_data_rows; + l_finished_flags ut_integer_list; + l_already_waited_for number(10,2) := 0; + l_finished boolean := false; + lc_init_wait_sec constant naturaln := coalesce(a_initial_timeout, 60 ); -- 1 minute + lc_max_wait_sec constant naturaln := coalesce(a_timeout_sec, 60 * 60 * 4); -- 4 hours + l_wait_for integer := lc_init_wait_sec; + lc_short_sleep_time constant number(1,1) := 0.1; --sleep for 100 ms between checks + lc_long_sleep_time constant number(1) := 1; --sleep for 1 s when waiting long + lc_long_wait_time constant number(1) := 1; --waiting more than 1 sec + l_sleep_time number(2,1) := lc_short_sleep_time; + lc_bulk_limit constant integer := 5000; + l_max_message_id integer := lc_bulk_limit; + + procedure remove_read_data(a_message_rowids t_rowid_tab) is + pragma autonomous_transaction; + begin + forall i in 1 .. a_message_rowids.count + delete from ut_output_clob_buffer_tmp a + where rowid = a_message_rowids(i); + commit; + end; + + procedure remove_buffer_info is + pragma autonomous_transaction; + begin + delete from ut_output_buffer_info_tmp a + where a.output_id = self.output_id; + commit; + end; + + begin + while not l_finished loop + with ordered_buffer as ( + select /*+ index(a) */ a.rowid, ut_output_data_row(a.text, a.item_type), is_finished + from ut_output_clob_buffer_tmp a + where a.output_id = self.output_id + and a.message_id <= l_max_message_id + order by a.message_id + ) + select b.* + bulk collect into l_message_rowids, l_buffer_data, l_finished_flags + from ordered_buffer b; + + --nothing fetched from output, wait and try again + if l_buffer_data.count = 0 then + $if dbms_db_version.version >= 18 $then + dbms_session.sleep(l_sleep_time); + $else + dbms_lock.sleep(l_sleep_time); + $end + l_already_waited_for := l_already_waited_for + l_sleep_time; + if l_already_waited_for > lc_long_wait_time then + l_sleep_time := lc_long_sleep_time; + end if; + else + --reset wait time + -- we wait lc_max_wait_sec for new message + l_wait_for := lc_max_wait_sec; + l_already_waited_for := 0; + l_sleep_time := lc_short_sleep_time; + for i in 1 .. l_buffer_data.count loop + if l_buffer_data(i).text is not null then + pipe row(l_buffer_data(i)); + elsif l_finished_flags(i) = 1 then + l_finished := true; + exit; + end if; + end loop; + remove_read_data(l_message_rowids); + l_max_message_id := l_max_message_id + lc_bulk_limit; + end if; + if l_finished or l_already_waited_for >= l_wait_for then + remove_buffer_info(); + if l_already_waited_for > 0 and l_already_waited_for >= l_wait_for then + raise_application_error( + ut_utils.gc_out_buffer_timeout, + 'Timeout occurred while waiting for output data. Waited for: '||l_already_waited_for||' seconds.' + ); + end if; + end if; + end loop; + return; + end; + + overriding member function get_lines_cursor(a_initial_timeout natural := null, a_timeout_sec natural := null) return sys_refcursor is + l_lines sys_refcursor; + begin + open l_lines for + select text, item_type + from table(self.get_lines(a_initial_timeout, a_timeout_sec)); + return l_lines; + end; + + overriding member procedure lines_to_dbms_output(self in ut_output_clob_table_buffer, a_initial_timeout natural := null, a_timeout_sec natural := null) is + l_data sys_refcursor; + l_clob clob; + l_item_type varchar2(32767); + l_lines ut_varchar2_list; + begin + l_data := self.get_lines_cursor(a_initial_timeout, a_timeout_sec); + loop + fetch l_data into l_clob, l_item_type; + exit when l_data%notfound; + l_lines := ut_utils.clob_to_table(l_clob); + for i in 1 .. l_lines.count loop + dbms_output.put_line(l_lines(i)); + end loop; + end loop; + close l_data; + end; + + member procedure cleanup_buffer(self in ut_output_clob_table_buffer, a_retention_time_sec natural := null) is + gc_buffer_retention_sec constant naturaln := coalesce(a_retention_time_sec, 60 * 60 * 24); -- 24 hours + l_retention_days number := gc_buffer_retention_sec / (60 * 60 * 24); + l_max_retention_date date := sysdate - l_retention_days; + pragma autonomous_transaction; + begin + delete from ut_output_clob_buffer_tmp t + where t.output_id + in (select i.output_id from ut_output_buffer_info_tmp i where i.start_date <= l_max_retention_date); + + delete from ut_output_buffer_info_tmp i where i.start_date <= l_max_retention_date; + commit; + end; + +end; +/ diff --git a/source/core/output_buffers/ut_output_clob_table_buffer.tps b/source/core/output_buffers/ut_output_clob_table_buffer.tps new file mode 100644 index 000000000..83e342106 --- /dev/null +++ b/source/core/output_buffers/ut_output_clob_table_buffer.tps @@ -0,0 +1,32 @@ +create or replace type ut_output_clob_table_buffer under ut_output_buffer_base ( + /* + utPLSQL - Version 3 + Copyright 2016 - 2018 utPLSQL Project + + Licensed under the Apache License, Version 2.0 (the "License"): + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + + start_date date, + last_message_id number(38,0), + constructor function ut_output_clob_table_buffer(self in out nocopy ut_output_clob_table_buffer, a_output_id raw := null) return self as result, + overriding member procedure init(self in out nocopy ut_output_clob_table_buffer), + overriding member procedure send_line(self in out nocopy ut_output_clob_table_buffer, a_text varchar2, a_item_type varchar2 := null), + overriding member procedure send_lines(self in out nocopy ut_output_clob_table_buffer, a_text_list ut_varchar2_rows, a_item_type varchar2 := null), + overriding member procedure send_clob(self in out nocopy ut_output_clob_table_buffer, a_text clob, a_item_type varchar2 := null), + overriding member procedure close(self in out nocopy ut_output_clob_table_buffer), + overriding member function get_lines(a_initial_timeout natural := null, a_timeout_sec natural := null) return ut_output_data_rows pipelined, + overriding member function get_lines_cursor(a_initial_timeout natural := null, a_timeout_sec natural := null) return sys_refcursor, + overriding member procedure lines_to_dbms_output(self in ut_output_clob_table_buffer, a_initial_timeout natural := null, a_timeout_sec natural := null), + member procedure cleanup_buffer(self in ut_output_clob_table_buffer, a_retention_time_sec natural := null) +) not final +/ diff --git a/source/core/output_buffers/ut_output_table_buffer.tpb b/source/core/output_buffers/ut_output_table_buffer.tpb index 5a9bdcce7..2b29e8ee5 100644 --- a/source/core/output_buffers/ut_output_table_buffer.tpb +++ b/source/core/output_buffers/ut_output_table_buffer.tpb @@ -52,11 +52,20 @@ create or replace type body ut_output_table_buffer is pragma autonomous_transaction; begin if a_text is not null or a_item_type is not null then - self.last_message_id := self.last_message_id + 1; - insert into ut_output_buffer_tmp(output_id, message_id, text, item_type) - values (self.output_id, self.last_message_id, a_text, a_item_type); + if length(a_text) > ut_utils.gc_max_storage_varchar2_len then + self.send_lines( + ut_utils.convert_collection( + ut_utils.clob_to_table(a_text, ut_utils.gc_max_storage_varchar2_len) + ), + a_item_type + ); + else + self.last_message_id := self.last_message_id + 1; + insert into ut_output_buffer_tmp(output_id, message_id, text, item_type) + values (self.output_id, self.last_message_id, a_text, a_item_type); + end if; + commit; end if; - commit; end; overriding member procedure send_lines(self in out nocopy ut_output_table_buffer, a_text_list ut_varchar2_rows, a_item_type varchar2 := null) is @@ -74,17 +83,25 @@ create or replace type body ut_output_table_buffer is pragma autonomous_transaction; begin if a_text is not null and a_text != empty_clob() or a_item_type is not null then - self.last_message_id := self.last_message_id + 1; - insert into ut_output_buffer_tmp(output_id, message_id, text, item_type) - values (self.output_id, self.last_message_id, a_text, a_item_type); + if length(a_text) > ut_utils.gc_max_storage_varchar2_len then + self.send_lines( + ut_utils.convert_collection( + ut_utils.clob_to_table(a_text, ut_utils.gc_max_storage_varchar2_len) + ), + a_item_type + ); + else + self.last_message_id := self.last_message_id + 1; + insert into ut_output_buffer_tmp(output_id, message_id, text, item_type) + values (self.output_id, self.last_message_id, a_text, a_item_type); + end if; + commit; end if; - commit; end; overriding member function get_lines(a_initial_timeout natural := null, a_timeout_sec natural := null) return ut_output_data_rows pipelined is - type t_rowid_tab is table of urowid; - l_message_rowids t_rowid_tab; - l_buffer_data ut_output_data_rows; + l_buffer_data ut_varchar2_rows; + l_item_types ut_varchar2_rows; l_finished_flags ut_integer_list; l_already_waited_for number(10,2) := 0; l_finished boolean := false; @@ -98,13 +115,25 @@ create or replace type body ut_output_table_buffer is lc_bulk_limit constant integer := 5000; l_max_message_id integer := lc_bulk_limit; - procedure remove_read_data(a_message_rowids t_rowid_tab) is + procedure get_data_from_buffer( + a_max_message_id integer, + a_buffer_data out nocopy ut_varchar2_rows, + a_item_types out nocopy ut_varchar2_rows, + a_finished_flags out nocopy ut_integer_list + ) is pragma autonomous_transaction; begin - forall i in 1 .. a_message_rowids.count - delete from ut_output_buffer_tmp a - where rowid = a_message_rowids(i); + delete from ( + select * + from ut_output_buffer_tmp o + where o.output_id = self.output_id + and o.message_id <= a_max_message_id + order by o.message_id + ) d + returning d.text, d.item_type, d.is_finished + bulk collect into a_buffer_data, a_item_types, a_finished_flags; commit; + end; procedure remove_buffer_info is @@ -117,17 +146,7 @@ create or replace type body ut_output_table_buffer is begin while not l_finished loop - with ordered_buffer as ( - select /*+ index(a) */ a.rowid, ut_output_data_row(a.text, a.item_type), is_finished - from ut_output_buffer_tmp a - where a.output_id = self.output_id - and a.message_id <= l_max_message_id - order by a.message_id - ) - select b.* - bulk collect into l_message_rowids, l_buffer_data, l_finished_flags - from ordered_buffer b; - + get_data_from_buffer( l_max_message_id, l_buffer_data, l_item_types, l_finished_flags); --nothing fetched from output, wait and try again if l_buffer_data.count = 0 then $if dbms_db_version.version >= 18 $then @@ -146,14 +165,13 @@ create or replace type body ut_output_table_buffer is l_already_waited_for := 0; l_sleep_time := lc_short_sleep_time; for i in 1 .. l_buffer_data.count loop - if l_buffer_data(i).text is not null then - pipe row(l_buffer_data(i)); + if l_buffer_data(i) is not null then + pipe row(ut_output_data_row(l_buffer_data(i),l_item_types(i))); elsif l_finished_flags(i) = 1 then l_finished := true; exit; end if; end loop; - remove_read_data(l_message_rowids); l_max_message_id := l_max_message_id + lc_bulk_limit; end if; if l_finished or l_already_waited_for >= l_wait_for then diff --git a/source/create_synonyms_and_grants_for_public.sql b/source/create_synonyms_and_grants_for_public.sql index dcc940737..609d78ccf 100644 --- a/source/create_synonyms_and_grants_for_public.sql +++ b/source/create_synonyms_and_grants_for_public.sql @@ -74,6 +74,7 @@ grant execute on &&ut3_owner..ut_coverage_options to public; grant execute on &&ut3_owner..ut_coverage_helper to public; grant execute on &&ut3_owner..ut_output_buffer_base to public; grant execute on &&ut3_owner..ut_output_table_buffer to public; +grant execute on &&ut3_owner..ut_output_clob_table_buffer to public; grant execute on &&ut3_owner..ut_file_mappings to public; grant execute on &&ut3_owner..ut_file_mapping to public; grant execute on &&ut3_owner..ut_file_mapper to public; @@ -154,6 +155,7 @@ create public synonym ut_coverage_options for &&ut3_owner..ut_coverage_options; create public synonym ut_coverage_helper for &&ut3_owner..ut_coverage_helper; create public synonym ut_output_buffer_base for &&ut3_owner..ut_output_buffer_base; create public synonym ut_output_table_buffer for &&ut3_owner..ut_output_table_buffer; +create public synonym ut_output_clob_table_buffer for &&ut3_owner..ut_output_clob_table_buffer; create public synonym ut_file_mappings for &&ut3_owner..ut_file_mappings; create public synonym ut_file_mapping for &&ut3_owner..ut_file_mapping; create public synonym ut_file_mapper for &&ut3_owner..ut_file_mapper; diff --git a/source/create_user_grants.sql b/source/create_user_grants.sql index feab90933..7aa5deb39 100644 --- a/source/create_user_grants.sql +++ b/source/create_user_grants.sql @@ -95,6 +95,7 @@ grant execute on &&ut3_owner..ut_output_buffer_base to &ut3_user; grant execute on &&ut3_owner..ut_output_data_row to &ut3_user; grant execute on &&ut3_owner..ut_output_data_rows to &ut3_user; grant execute on &&ut3_owner..ut_output_table_buffer to &ut3_user; +grant execute on &&ut3_owner..ut_output_clob_table_buffer to &ut3_user; grant execute on &&ut3_owner..ut_file_mappings to &ut3_user; grant execute on &&ut3_owner..ut_file_mapping to &ut3_user; grant execute on &&ut3_owner..ut_file_mapper to &ut3_user; diff --git a/source/create_user_synonyms.sql b/source/create_user_synonyms.sql index b08cb7b0e..cb2f7ab67 100644 --- a/source/create_user_synonyms.sql +++ b/source/create_user_synonyms.sql @@ -97,6 +97,7 @@ create or replace synonym &ut3_user..ut_coverage_options for &&ut3_owner..ut_cov create or replace synonym &ut3_user..ut_coverage_helper for &&ut3_owner..ut_coverage_helper; create or replace synonym &ut3_user..ut_output_buffer_base for &&ut3_owner..ut_output_buffer_base; create or replace synonym &ut3_user..ut_output_table_buffer for &&ut3_owner..ut_output_table_buffer; +create or replace synonym &ut3_user..ut_output_clob_table_buffer for &&ut3_owner..ut_output_clob_table_buffer; create or replace synonym &ut3_user..ut_file_mappings for &&ut3_owner..ut_file_mappings; create or replace synonym &ut3_user..ut_file_mapping for &&ut3_owner..ut_file_mapping; create or replace synonym &ut3_user..ut_file_mapper for &&ut3_owner..ut_file_mapper; diff --git a/source/install.sql b/source/install.sql index 08a5b59d7..728f4349f 100644 --- a/source/install.sql +++ b/source/install.sql @@ -99,9 +99,12 @@ alter session set current_schema = &&ut3_owner; --output buffer table @@install_component.sql 'core/output_buffers/ut_output_buffer_info_tmp.sql' @@install_component.sql 'core/output_buffers/ut_output_buffer_tmp.sql' +@@install_component.sql 'core/output_buffers/ut_output_clob_buffer_tmp.sql' --output buffer table api @@install_component.sql 'core/output_buffers/ut_output_table_buffer.tps' @@install_component.sql 'core/output_buffers/ut_output_table_buffer.tpb' +@@install_component.sql 'core/output_buffers/ut_output_clob_table_buffer.tps' +@@install_component.sql 'core/output_buffers/ut_output_clob_table_buffer.tpb' @@install_component.sql 'core/types/ut_output_reporter_base.tps' diff --git a/source/reporters/ut_debug_reporter.tpb b/source/reporters/ut_debug_reporter.tpb index 186290296..f22e8748b 100644 --- a/source/reporters/ut_debug_reporter.tpb +++ b/source/reporters/ut_debug_reporter.tpb @@ -18,7 +18,7 @@ create or replace type body ut_debug_reporter is constructor function ut_debug_reporter(self in out nocopy ut_debug_reporter) return self as result is begin - self.init($$plsql_unit); + self.init($$plsql_unit,ut_output_clob_table_buffer()); self.start_time := current_timestamp(); self.event_time := current_timestamp(); return; diff --git a/source/reporters/ut_realtime_reporter.tpb b/source/reporters/ut_realtime_reporter.tpb index e712c8f85..7c674cc7d 100644 --- a/source/reporters/ut_realtime_reporter.tpb +++ b/source/reporters/ut_realtime_reporter.tpb @@ -20,7 +20,7 @@ create or replace type body ut_realtime_reporter is self in out nocopy ut_realtime_reporter ) return self as result is begin - self.init($$plsql_unit); + self.init($$plsql_unit,ut_output_clob_table_buffer()); total_number_of_tests := 0; current_test_number := 0; current_indent := 0; diff --git a/source/uninstall_objects.sql b/source/uninstall_objects.sql index ed84aa0a0..3599549a3 100644 --- a/source/uninstall_objects.sql +++ b/source/uninstall_objects.sql @@ -247,12 +247,18 @@ drop type ut_suite_item force; drop type ut_output_table_buffer force; +drop type ut_output_clob_table_buffer force; + drop type ut_output_buffer_base force; drop view ut_output_buffer_tmp; drop table ut_output_buffer_tmp$ purge; +drop view ut_output_clob_buffer_tmp; + +drop table ut_output_clob_buffer_tmp$ purge; + drop view ut_output_buffer_info_tmp; drop table ut_output_buffer_info_tmp$; diff --git a/test/core/reporters/test_realtime_reporter.pkb b/test/core/reporters/test_realtime_reporter.pkb index c4ceb9d5d..2267e9c41 100644 --- a/test/core/reporters/test_realtime_reporter.pkb +++ b/test/core/reporters/test_realtime_reporter.pkb @@ -112,7 +112,7 @@ create or replace package body test_realtime_reporter as -- consume select test_event_object(item_type, xmltype(text)) bulk collect into g_events - from table(ut3.ut_output_table_buffer(l_reporter.output_buffer.output_id).get_lines()) + from table(l_reporter.get_lines()) where trim(text) is not null and item_type is not null; end run_report_and_cache_result; end create_test_suites_and_run; diff --git a/test/core/test_output_buffer.pkb b/test/core/test_output_buffer.pkb index 848824ea8..79e88f8d8 100644 --- a/test/core/test_output_buffer.pkb +++ b/test/core/test_output_buffer.pkb @@ -9,7 +9,7 @@ create or replace package body test_output_buffer is l_buffer ut3.ut_output_buffer_base; begin --Arrange - l_buffer := ut3.ut_output_table_buffer(); + l_buffer := ut3.ut_output_clob_table_buffer(); l_expected_text := to_clob(lpad('a text', 31000, ',a text')) || chr(10) || to_clob(lpad('a text', 31000, ',a text')) || chr(13) || to_clob(lpad('a text', 31000, ',a text'))