Application Cleanup

Using the `irods.at_client_exit` module, we may register user-defined functions to be executed at or around the

time when the Python iRODS Client is engaged in object teardown (also called "cleanup") operations.

This is analogous to Python's [atexit module](https://docs.python.org/3/library/atexit.html#module-atexit),

except that here we have the extra resolution to specify that a function or callable object be expressly before,

or expressly after, aforementioned object teardown stage:

from irods import at_client_exit

at_client_exit.register_for_execution_after_prc_cleanup(lambda: print("PRC cleanup has completed."))

at_client_exit.register_for_execution_before_prc_cleanup(lambda: print("PRC cleanup is about to start."))

A function normally cannot be registered multiple times to run in the same stage, but we may overcome this limitation

(and, optionally, arguments set for the invocation) by wrapping the same function into two different callable objects:

def print_exit_message(n):

print(f"Called just after PRC cleanup - iteration {n}")

for n_iter in (1,2):

at_client_exit.register_for_execution_after_prc_cleanup(

at_client_exit.unique_function_invocation(print_exit_message, tup_args = (n_iter,))

)

The output of the above, upon script exit, will be:

Called just after PRC cleanup - iteration 2

Called just after PRC cleanup - iteration 1

which may be reversed from the order that one might expect. This is because -- similarly as with Python atexit module, and

consistently with the teardown of higher abstractions before lower ones -- functions _registered_ later within a given cleanup

stage will actually be _executed_ sooner (i.e. in "LIFO" order).

import atexit

import enum

import logging

import sys

import threading

logger = logging.getLogger(__name__)

class LibraryCleanupStage(enum.Enum):

# Integer value determines relative order of execution among the stages.

BEFORE = 10

DURING = 20

AFTER = 30

def __lt__(self, other):

return self.value < other.value

def NOTIFY_VIA_ATTRIBUTE(func, stage):

func.at_exit_stage = stage

initialization_lock = threading.Lock()

initialized = False

_stage_notify = {}

def _register(stage_for_execution, function, stage_notify_function = None):

with initialization_lock:

global initialized

if not initialized:

initialized = True

atexit.register(_call_cleanup_functions)

array = _cleanup_functions[stage_for_execution]

if function not in array:

array.append(function)

_stage_notify[function] = stage_notify_function

def _reset_cleanup_functions():

global _cleanup_functions

_cleanup_functions = {LibraryCleanupStage.BEFORE:[],

LibraryCleanupStage.DURING:[],

LibraryCleanupStage.AFTER:[]

}

_reset_cleanup_functions()

def _call_cleanup_functions():

ordered_funclists = sorted((pri,fl) for (pri,fl) in _cleanup_functions.items())

try:

# Run each cleanup stage.

for stage,function_list in ordered_funclists:

# Within each cleanup stage, run all registered functions last-in, first-out. (Per atexit conventions.)

for function in reversed(function_list):

# Ensure we execute all cleanup functions regardless of some of them raising exceptions.

try:

notify = _stage_notify.get(function)

if notify:

notify(function, stage)

function()

except Exception as exc:

logger.warning("%r raised from %s", exc, function)

finally:

_reset_cleanup_functions()

def register_for_execution_before_prc_cleanup(function, **kw_args):

"""Register a function to be called by the Python iRODS Client (PRC) at exit time.

return _register(LibraryCleanupStage.BEFORE, function, **kw_args)

def register_for_execution_after_prc_cleanup(function, **kw_args):

"""Register a function to be called by the Python iRODS Client (PRC) at exit time.

return _register(LibraryCleanupStage.AFTER, function, **kw_args)

class unique_function_invocation:

"""Wrap a function object to provide a unique handle for registration of a given

def __init__(self, func, tup_args = (), kw_args = ()):

self.tup_args = tup_args

self.kw_args = dict(kw_args)

self.func = func

self.at_exit_stage = None

def __call__(self):

return (self.func)(*self.tup_args, **self.kw_args)

def get_stage(n = 2):

"""A utility function that can be called from within the highest call frame (use a higher

caller = sys._getframe(n).f_locals.get('self')

if isinstance(caller, unique_function_invocation):

return caller.at_exit_stage

from . import at_client_exit

for fd in list((_fds or {}).keys()):

for ses in (_sessions or []).copy():

with _sessions_lock:

at_client_exit._register(

at_client_exit.LibraryCleanupStage.DURING,

_cleanup_remaining_sessions)

import os

import re

import subprocess

import sys

import textwrap

import unittest

import irods.at_client_exit

import irods.test.modules as test_modules

class TestCleanupFunctions(unittest.TestCase):

def test_execution_of_client_exit_functions_at_proper_time__issue_614(self):

helper_script = os.path.join(test_modules.__path__[0], 'test_client_exit_functions.py')

# Note: The enum.Enum subclass's __members__ is an ordered dictionary, i.e. key order is preserved:

# https://docs.python.org/3.6/library/enum.html#iteration

# This is essential for the test to pass, since:

# - We assume that key order to be preserved in the rendering of the arguments list presented to the test helper script.

# - That argument order determines the below-asserted order of execution, at script exit, of the functions registered therein.

args = [""] + list(irods.at_client_exit.LibraryCleanupStage.__members__)

p = subprocess.Popen([sys.executable, helper_script, *args], stdout=subprocess.PIPE)

script_output = p.communicate()[0].decode().strip()

from irods.test.modules.test_client_exit_functions import projected_output_from_innate_list_order

self.assertEqual(

projected_output_from_innate_list_order(args),

script_output

)

def test_that_client_exit_functions_execute_in_LIFO_order_and_despite_uncaught_exceptions__issue_614(self):

process = subprocess.Popen([sys.executable, '-c',

textwrap.dedent("""

""")], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

stdout_content, stderr_content = process.communicate()

self.assertEqual(len(list(re.finditer(br'ZeroDivisionError.*\n',stderr_content))), 4)

self.assertEqual(b','.join([m.group() for m in re.finditer(br'(\w+)',stdout_content)]),

b'before,during,after2,after1')

def test_notifying_client_exit_functions_of_stage_in_which_they_are_called__issue_614(self):

process = subprocess.Popen([sys.executable, '-c',

textwrap.dedent("""

""")], stdout=subprocess.PIPE)

stdout_content, _ = process.communicate()

stdout_lines = stdout_content.split(b"\n")

self.assertIn(b"before:BEFORE", stdout_lines)

self.assertIn(b"after:AFTER", stdout_lines)

if __name__ == '__main__':

# let the tests find the parent irods lib

sys.path.insert(0, os.path.abspath('../..'))

unittest.main()

import irods.at_client_exit

import sys

def get_stage_object_and_value_from_name(stage_name):

try:

obj = getattr(irods.at_client_exit.LibraryCleanupStage, stage_name)

return obj, obj.value

except AttributeError:

return None, stage_name

def object_printer(name, stream = sys.stdout):

(obj, _) = get_stage_object_and_value_from_name(stage_name = name)

return lambda: print(f'[{name if not obj else obj.value}]', end='', file = stream)

def projected_output_from_innate_list_order(name_list):

import io

in_memory_stream = io.StringIO()

for name in name_list:

func = object_printer(name, stream = in_memory_stream)

func()

return in_memory_stream.getvalue()

if __name__ == '__main__':

function_info = [[*get_stage_object_and_value_from_name(name),object_printer(name)] for name in reversed(sys.argv[1:])]

# For each argument to the script that represents a cleanup stage: register a function to print the stage's value, to be run during that stage.

for key in function_info.copy():

(stage,name,func) = key

if not stage:

continue

irods.at_client_exit._register(stage, func)

function_info.remove(key)

# Immediately execute any leftover functions, since the test expects the corresponding output to precede that of their counterparts run during cleanup.

for stage, name, func in function_info:

func()