Updated as per code review comments, added comments on things to do.

jgebal · jgebal · commit 40bc0854ce8a · 2016-11-16T01:31:47.000Z
diff --git a/source/core/types/ut_output_dbms_pipe.tpb b/source/core/types/ut_output_dbms_pipe.tpb
@@ -28,12 +28,12 @@ create or replace type body ut_output_dbms_pipe as
       i := i + c_size_limit_chars;
     end loop;
     --SEND is closed by a EOM message
-    ut_output_pipe_helper.send( self.output_id, ut_output_pipe_helper.gc_output_eom );
+    ut_output_pipe_helper.send_eom(self.output_id);
   end;
 
   overriding member procedure close(self in out nocopy ut_output_dbms_pipe) is
   begin
-    ut_output_pipe_helper.send(self.output_id, ut_output_pipe_helper.gc_output_eot);
+    ut_output_pipe_helper.send_eot(self.output_id);
     ut_output_pipe_helper.flush(self.output_id);
   end;
 
@@ -43,6 +43,7 @@ create or replace type body ut_output_dbms_pipe as
     l_text            clob;
     l_timeout_occured boolean;
     l_need_to_send    boolean;
+    l_item_type       integer;
   begin
     if a_output_id is not null then
 
@@ -51,20 +52,26 @@ create or replace type body ut_output_dbms_pipe as
         l_need_to_send := false;
 
         --build a row. As a rule one call to 'send' is one row.
+        -- TODO - add detection of newline so that for each new line recieved in text, the code generates a new output row
         loop
-          l_timeout_occured := dbms_pipe.receive_message(a_output_id, a_timeout_sec) != 0;
+          dbms_pipe.reset_buffer;
+          l_timeout_occured := (dbms_pipe.receive_message(a_output_id, a_timeout_sec) != 0);
           exit when l_timeout_occured;
+          l_item_type := dbms_pipe.next_item_type();
+          exit when l_item_type in (ut_output_pipe_helper.gc_eom, ut_output_pipe_helper.gc_eot);
           dbms_pipe.unpack_message(l_text_part);
-          exit when ( l_text_part in (ut_output_pipe_helper.gc_output_eom, ut_output_pipe_helper.gc_output_eot) );
-          dbms_lob.writeappend(l_text, length(l_text_part), l_text_part);
+          if l_text_part is not null then
+            dbms_lob.writeappend(l_text, length(l_text_part), l_text_part);
+          end if;
           l_need_to_send := true;
         end loop;
 
         if l_need_to_send then
           pipe row( l_text );
+          dbms_lob.freetemporary(l_text);
         end if;
 
-        exit when (l_timeout_occured or l_text_part = ut_output_pipe_helper.gc_output_eot);
+        exit when (l_timeout_occured or l_item_type = ut_output_pipe_helper.gc_eot);
       end loop;
 
       l_flag := dbms_pipe.remove_pipe(a_output_id);
diff --git a/source/core/ut_output_pipe_helper.pkb b/source/core/ut_output_pipe_helper.pkb
@@ -1,8 +1,11 @@
 create or replace package body ut_output_pipe_helper is
 
   -- private
-
-  type tt_pipe_buffer is table of varchar2(4000);
+  type t_pipe_buffer_rec is record(
+    message_type integer,
+    message      varchar2(4000)
+  );
+  type tt_pipe_buffer is table of t_pipe_buffer_rec;
 
   type t_output_buffer is record(
     to_flush boolean := false,
@@ -26,7 +29,13 @@ create or replace package body ut_output_pipe_helper is
       --iterate through buffer and try to sent all data from buffer
       -- exit loop on timeout
       while i is not null loop
-        dbms_pipe.pack_message( a_buffer(i) );
+        if a_buffer(i).message_type = gc_eom then
+          dbms_pipe.pack_message_rowid( NULL );
+        elsif a_buffer(i).message_type = gc_eot then
+          dbms_pipe.pack_message_raw( NULL );
+        else
+          dbms_pipe.pack_message( a_buffer(i).message );
+        end if;
         l_is_successful := dbms_pipe.send_message(a_output_id, a_timeout_seconds) = 0;
         exit when not l_is_successful;
         l_last_item := i;
@@ -57,6 +66,7 @@ create or replace package body ut_output_pipe_helper is
   --iterates through all the output buffers that were not purged
   --for each buffer tries to send the content of the buffer
   --if all content was sent, buffer is recycled (removed)
+  --returns true if the flushing succeeded for all buffers
   function flush_buffers(a_timeout_seconds naturaln := 0) return boolean is
     l_output_id           t_output_id;
     l_output_id_to_delete t_output_id;
@@ -66,10 +76,10 @@ create or replace package body ut_output_pipe_helper is
     l_output_id := g_outputs_buffer.first;
     while l_output_id is not null loop
       l_output_id_to_delete := null;
-      if send_from_buffer(l_output_id, a_timeout_seconds) = true then
+
+      l_timed_out := not send_from_buffer(l_output_id, a_timeout_seconds);
+      if not l_timed_out then
         l_output_id_to_delete := l_output_id;
-      else
-        l_timed_out := true;
       end if;
       l_output_id := g_outputs_buffer.next(l_output_id);
 
@@ -95,27 +105,33 @@ create or replace package body ut_output_pipe_helper is
     return true;
   end;
 
-  --remove all the pipes anf purges all the buffers
+  -- - remove pipes associated with buffers that are not yet deleted
+  -- - delete all the buffers
+  -- TODO - The purge procedure needs to be called by top level program (ut_runner)
+  --  in the EXCEPTION WHEN OTHERS block before raising back,
+  --  so that it tries to remove pipes on any exception before raising back to caller
   procedure purge is
     l_pipe_removal_status integer;
     l_output_id           t_output_id;
   begin
     l_output_id := g_outputs_buffer.first;
     while l_output_id is not null loop
+      l_pipe_removal_status := dbms_pipe.remove_pipe(l_output_id);
       l_output_id := g_outputs_buffer.next(l_output_id);
     end loop;
     g_outputs_buffer.delete;
   end;
 
   --writes the message to the end of the buffer
-  procedure buffer(a_output_id t_output_id, a_text t_pipe_item) is
+  procedure buffer(a_output_id t_output_id, a_text t_pipe_item, a_message_type integer := null) is
   begin
     if not g_outputs_buffer.exists(a_output_id) then
       g_outputs_buffer(a_output_id).data := tt_pipe_buffer();
       g_outputs_buffer(a_output_id).to_flush := false;
     end if;
     g_outputs_buffer(a_output_id).data.extend;
-    g_outputs_buffer(a_output_id).data(g_outputs_buffer(a_output_id).data.last) :=  a_text;
+    g_outputs_buffer(a_output_id).data(g_outputs_buffer(a_output_id).data.last).message_type :=  a_message_type;
+    g_outputs_buffer(a_output_id).data(g_outputs_buffer(a_output_id).data.last).message :=  a_text;
   end;
 
 
@@ -131,10 +147,27 @@ create or replace package body ut_output_pipe_helper is
       l_is_successful := send_from_buffer(a_output_id);
   end;
 
+  --sends a end of message into a a pipe
+  procedure send_eom(a_output_id t_output_id) is
+    l_is_successful boolean;
+  begin
+      buffer(a_output_id, null, gc_eom);
+      l_is_successful := send_from_buffer(a_output_id);
+  end;
+
+  --sends a end of message into a a pipe
+  procedure send_eot(a_output_id t_output_id) is
+    l_is_successful boolean;
+  begin
+      buffer(a_output_id, null, gc_eot);
+      l_is_successful := send_from_buffer(a_output_id);
+  end;
+
   --marks a buffer as ready to be flushed
   --tries to flush all the data from all the outputs buffers to the pipes immediately
   --in case, all buffers outputs are to be flused, it will try with a timeout.
   procedure flush(a_output_id t_output_id, a_timeout_seconds naturaln := gc_flush_timeout_seconds) is
+    l_buffers_flushed boolean := false;
   begin
     if g_outputs_buffer.exists(a_output_id) then
       g_outputs_buffer(a_output_id).to_flush := true;
@@ -144,9 +177,16 @@ create or replace package body ut_output_pipe_helper is
       --try as many times as there are seconds for timeout
       --each time try with one second delay
       for i in 1 .. a_timeout_seconds loop
-        exit when flush_buffers(a_timeout_seconds => 1) = true;
+        l_buffers_flushed := flush_buffers(a_timeout_seconds => 1);
+        exit when l_buffers_flushed;
       end loop;
-      purge();
+
+      --if timeout occured and buffers were not flushed then
+      -- - remove pipes associated with non flushed buffers
+      -- - delete the buffers
+      if not l_buffers_flushed then
+        purge();
+      end if;
     end if;
 
   end;
diff --git a/source/core/ut_output_pipe_helper.pks b/source/core/ut_output_pipe_helper.pks
@@ -9,14 +9,21 @@ create or replace package ut_output_pipe_helper is
 
   --decides how long the process will be waiting to flush buffers
   gc_flush_timeout_seconds constant natural := 60;
-  gc_output_eom            constant varchar2(30) := '[{-end-of-message-}]';
-  gc_output_eot            constant varchar2(30) := '[{-end-of-transmission-}]';
+  gc_text                  constant integer := 9;
+  gc_eom                   constant integer := 11;
+  gc_eot                   constant integer := 23;
 
   --adds message to pipe buffer and tries to sent all messages from the buffer
   --exists immediately when sending timesout (pipe full)
   --the messages that were sent are removed from buffer
   procedure send(a_output_id t_output_id, a_text t_pipe_item);
 
+  --sends a end of message into a a pipe
+  procedure send_eom(a_output_id t_output_id);
+
+  --sends a end of message into a a pipe
+  procedure send_eot(a_output_id t_output_id);
+
   --marks a buffer as ready to be flushed and then
   --tries to flush all the data from all the outputs buffers to the pipes
   --in case, all buffers outputs are to be flushed, it will try until a timeout occurs.
