From e0073ea1adf5c189e2d169c86f7a5a55aa316733 Mon Sep 17 00:00:00 2001 From: Zak V Date: Sat, 3 Oct 2020 13:48:15 -0400 Subject: [PATCH 01/10] Added support for saving git repo info. --- labscript/labscript.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/labscript/labscript.py b/labscript/labscript.py index 39fd574..2a7aa36 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -2206,6 +2206,14 @@ def save_labscripts(hdf5_file): info, err = process.communicate() if info or err: hdf5_file[save_path].attrs['hg ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') + if compiler.save_git_info: + module_filename = os.path.split(path)[1] + git_commands = [['branch', '--show-current'], ['rev-parse', '--verify', 'HEAD'], ['diff', 'HEAD', module_filename]] + for command in git_commands: + process = subprocess.Popen(['git'] + command, cwd=os.path.split(path)[0], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, startupinfo=startupinfo) + info, err = process.communicate() + hdf5_file[save_path].attrs['git ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') except ImportError: pass except WindowsError if os.name == 'nt' else None: @@ -2540,6 +2548,7 @@ def labscript_cleanup(): compiler.time_markers = {} compiler._PrimaryBLACS = None compiler.save_hg_info = True + compiler.save_git_info = True compiler.shot_properties = {} class compiler(object): @@ -2560,6 +2569,7 @@ class compiler(object): time_markers = {} _PrimaryBLACS = None save_hg_info = True + save_git_info = True shot_properties = {} # safety measure in case cleanup is called before init From df4e17f8ea00b3297a35ec938ba9948566b089b1 Mon Sep 17 00:00:00 2001 From: Zak V Date: Sat, 3 Oct 2020 14:35:59 -0400 Subject: [PATCH 02/10] Added support for save_hg_info and save_git_info options in the labconfig [labscript] section. --- labscript/labscript.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/labscript/labscript.py b/labscript/labscript.py index 2a7aa36..737b041 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -31,6 +31,7 @@ import labscript_utils.h5_lock, h5py import labscript_utils.properties +from labscript_utils.labconfig import LabConfig # This imports the default Qt library that other labscript suite code will # import as well, since it all uses qtutils. By having a Qt library already @@ -2547,8 +2548,8 @@ def labscript_cleanup(): compiler.wait_delay = 0 compiler.time_markers = {} compiler._PrimaryBLACS = None - compiler.save_hg_info = True - compiler.save_git_info = True + compiler.save_hg_info = LabConfig().getboolean('labscript', 'save_hg_info', fallback=True) + compiler.save_git_info = LabConfig().getboolean('labscript', 'save_git_info', fallback=False) compiler.shot_properties = {} class compiler(object): @@ -2568,8 +2569,8 @@ class compiler(object): wait_delay = 0 time_markers = {} _PrimaryBLACS = None - save_hg_info = True - save_git_info = True + save_hg_info = LabConfig().getboolean('labscript', 'save_hg_info', fallback=True) + save_git_info = LabConfig().getboolean('labscript', 'save_git_info', fallback=False) shot_properties = {} # safety measure in case cleanup is called before init From 58ed6db264c2ecd3f283bd42df845a83ef5b0c8f Mon Sep 17 00:00:00 2001 From: Zak V Date: Sun, 4 Oct 2020 12:28:35 -0400 Subject: [PATCH 03/10] save_labscripts()'s calls to git/hg for a given file now run in parallel. --- labscript/labscript.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/labscript/labscript.py b/labscript/labscript.py index 737b041..1d891e5 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -2201,18 +2201,24 @@ def save_labscripts(hdf5_file): hdf5_file.create_dataset(save_path, data=open(path).read()) if compiler.save_hg_info: hg_commands = [['log', '--limit', '1'], ['status'], ['diff']] + process_list = [] for command in hg_commands: process = subprocess.Popen(['hg'] + command + [os.path.split(path)[1]], cwd=os.path.split(path)[0], stdout=subprocess.PIPE, stderr=subprocess.PIPE, startupinfo=startupinfo) + process_list.append(process) + for process, command in zip(process_list, hg_commands): info, err = process.communicate() if info or err: hdf5_file[save_path].attrs['hg ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') if compiler.save_git_info: module_filename = os.path.split(path)[1] git_commands = [['branch', '--show-current'], ['rev-parse', '--verify', 'HEAD'], ['diff', 'HEAD', module_filename]] + process_list = [] for command in git_commands: process = subprocess.Popen(['git'] + command, cwd=os.path.split(path)[0], stdout=subprocess.PIPE, stderr=subprocess.PIPE, startupinfo=startupinfo) + process_list.append(process) + for process, command in zip(process_list, git_commands): info, err = process.communicate() hdf5_file[save_path].attrs['git ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') except ImportError: From 9a20221ea008fc7394f07d472f16e08b26fa6d0e Mon Sep 17 00:00:00 2001 From: Zak V Date: Sun, 4 Oct 2020 12:30:44 -0400 Subject: [PATCH 04/10] Removed "--verify" from git rev-parse call. --- labscript/labscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript/labscript.py b/labscript/labscript.py index 1d891e5..15b7d13 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -2212,7 +2212,7 @@ def save_labscripts(hdf5_file): hdf5_file[save_path].attrs['hg ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') if compiler.save_git_info: module_filename = os.path.split(path)[1] - git_commands = [['branch', '--show-current'], ['rev-parse', '--verify', 'HEAD'], ['diff', 'HEAD', module_filename]] + git_commands = [['branch', '--show-current'], ['rev-parse', 'HEAD'], ['diff', 'HEAD', module_filename]] process_list = [] for command in git_commands: process = subprocess.Popen(['git'] + command, cwd=os.path.split(path)[0], stdout=subprocess.PIPE, From dc8d917a87e0fe5e6dac351a4cc1eaee3d84514a Mon Sep 17 00:00:00 2001 From: Zak V Date: Sun, 4 Oct 2020 12:35:39 -0400 Subject: [PATCH 05/10] Added "git describe --always HEAD" to git commands. --- labscript/labscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript/labscript.py b/labscript/labscript.py index 15b7d13..cacedf7 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -2212,7 +2212,7 @@ def save_labscripts(hdf5_file): hdf5_file[save_path].attrs['hg ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') if compiler.save_git_info: module_filename = os.path.split(path)[1] - git_commands = [['branch', '--show-current'], ['rev-parse', 'HEAD'], ['diff', 'HEAD', module_filename]] + git_commands = [['branch', '--show-current'], ['describe', '--always', 'HEAD'], ['rev-parse', 'HEAD'], ['diff', 'HEAD', module_filename]] process_list = [] for command in git_commands: process = subprocess.Popen(['git'] + command, cwd=os.path.split(path)[0], stdout=subprocess.PIPE, From 2025755da06f08503d26719197b0f3cac99cca2f Mon Sep 17 00:00:00 2001 From: Zak V Date: Sun, 4 Oct 2020 12:38:10 -0400 Subject: [PATCH 06/10] Updated git/hg error message. --- labscript/labscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript/labscript.py b/labscript/labscript.py index cacedf7..3eaf3be 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -2224,7 +2224,7 @@ def save_labscripts(hdf5_file): except ImportError: pass except WindowsError if os.name == 'nt' else None: - sys.stderr.write('Warning: Cannot save Mercurial data for imported scripts. Check that the hg command can be run from the command line.\n') + sys.stderr.write('Warning: Cannot save version control data for imported scripts. Check that the hg and/or git command can be run from the command line.\n') def write_device_properties(hdf5_file): From 6022d97b9d7b2ec09f341edb21fe6fba4ea1cc66 Mon Sep 17 00:00:00 2001 From: Zak V Date: Sun, 4 Oct 2020 12:52:10 -0400 Subject: [PATCH 07/10] Added '--tags' to call to git describe. --- labscript/labscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript/labscript.py b/labscript/labscript.py index 3eaf3be..a8d1752 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -2212,7 +2212,7 @@ def save_labscripts(hdf5_file): hdf5_file[save_path].attrs['hg ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') if compiler.save_git_info: module_filename = os.path.split(path)[1] - git_commands = [['branch', '--show-current'], ['describe', '--always', 'HEAD'], ['rev-parse', 'HEAD'], ['diff', 'HEAD', module_filename]] + git_commands = [['branch', '--show-current'], ['describe', '--tags', '--always', 'HEAD'], ['rev-parse', 'HEAD'], ['diff', 'HEAD', module_filename]] process_list = [] for command in git_commands: process = subprocess.Popen(['git'] + command, cwd=os.path.split(path)[0], stdout=subprocess.PIPE, From 0f3f187aeeb3125de6ee959e9c76388d85499695 Mon Sep 17 00:00:00 2001 From: Zak V Date: Tue, 13 Oct 2020 02:01:49 -0400 Subject: [PATCH 08/10] Results from git/hg are now cached for faster shot compilation. --- labscript/labscript.py | 131 ++++++++++++++++++++++++++++++++--------- 1 file changed, 102 insertions(+), 29 deletions(-) diff --git a/labscript/labscript.py b/labscript/labscript.py index a8d1752..5337752 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -32,6 +32,7 @@ import labscript_utils.h5_lock, h5py import labscript_utils.properties from labscript_utils.labconfig import LabConfig +from labscript_utils.filewatcher import FileWatcher # This imports the default Qt library that other labscript suite code will # import as well, since it all uses qtutils. By having a Qt library already @@ -70,7 +71,10 @@ startupinfo.dwFlags |= 1 #subprocess.STARTF_USESHOWWINDOW # This variable isn't defined, but apparently it's equal to one. else: startupinfo = None - + +# Extract settings from labconfig +_SAVE_HG_INFO = LabConfig().getboolean('labscript', 'save_hg_info', fallback=True) +_SAVE_GIT_INFO = LabConfig().getboolean('labscript', 'save_git_info', fallback=False) class config(object): suppress_mild_warnings = True @@ -2174,8 +2178,88 @@ def generate_connection_table(hdf5_file): else: master_pseudoclock_name = compiler.master_pseudoclock.name dataset.attrs['master_pseudoclock'] = master_pseudoclock_name - - + +# Create a dictionary for caching results from vcs commands. The keys will be +# the paths to files that are saved during save_labscripts(). The values will be +# a list of tuples of the form (command, info, err); see the "Returns" section +# of the _run_vcs_commands() docstring for more info. Also create a FileWatcher +# instance for tracking when vcs results need updating. The callback will +# replace the outdated cache entry with a new list of updated vcs commands and +# outputs. +_vcs_cache = {} +def _file_watcher_callback(name, info, event): + _vcs_cache[name] = _run_vcs_commands(name) + +_file_watcher = FileWatcher(_file_watcher_callback) + +def _run_vcs_commands(path): + """Run some VCS commands on a file and return their output. + + The function is used to gather up version control system information so that + it can be stored in the hdf5 files of shots. This is for convenience and + compliments the full copy of the file already included in the shot file. + + Whether hg and git commands are run is controlled by the `save_hg_info` + and `save_git_info` options in the `[labscript]` section of the labconfig. + + Args: + path (str): The path with file name and extension of the file on which + the commands will be run. The working directory will be set to the + directory containing the specified file. + + Returns: + results (list of (tuple, str, str)): A list of tuples, each + containing information related to one vcs command of the form + (command, info, err). The first entry in that tuple is itself a + tuple of strings which was passed to subprocess.Popen() in order to + run the command. Then info is a string that contains the text + printed to stdout by that command, and err contains the text printed + to stderr by the command. + """ + # Gather together a list of commands to run. + module_directory, module_filename = os.path.split(path) + vcs_commands = [] + if compiler.save_hg_info: + hg_commands = [ + ['log', '--limit', '1'], + ['status'], + ['diff'], + ] + for command in hg_commands: + command = tuple(['hg'] + command + [module_filename]) + vcs_commands.append((command, module_directory)) + if compiler.save_git_info: + git_commands = [ + ['branch', '--show-current'], + ['describe', '--tags', '--always', 'HEAD'], + ['rev-parse', 'HEAD'], + ['diff', 'HEAD', module_filename], + ] + for command in git_commands: + command = tuple(['git'] + command) + vcs_commands.append((command, module_directory)) + + # Now go through and start running the commands. + process_list = [] + for command, module_directory in vcs_commands: + process = subprocess.Popen( + command, + cwd=module_directory, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + startupinfo=startupinfo, + ) + process_list.append((command, process)) + + # Gather up results from the commands issued. + results = [] + for command, process in process_list: + info, err = process.communicate() + info = info.decode('utf-8') + err = err.decode('utf-8') + results.append((command, info, err)) + return results + def save_labscripts(hdf5_file): if compiler.labscript_file is not None: script_text = open(compiler.labscript_file).read() @@ -2199,28 +2283,17 @@ def save_labscripts(hdf5_file): # Doesn't seem to want to double count files if you just import the contents of a file within a module continue hdf5_file.create_dataset(save_path, data=open(path).read()) - if compiler.save_hg_info: - hg_commands = [['log', '--limit', '1'], ['status'], ['diff']] - process_list = [] - for command in hg_commands: - process = subprocess.Popen(['hg'] + command + [os.path.split(path)[1]], cwd=os.path.split(path)[0], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, startupinfo=startupinfo) - process_list.append(process) - for process, command in zip(process_list, hg_commands): - info, err = process.communicate() - if info or err: - hdf5_file[save_path].attrs['hg ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') - if compiler.save_git_info: - module_filename = os.path.split(path)[1] - git_commands = [['branch', '--show-current'], ['describe', '--tags', '--always', 'HEAD'], ['rev-parse', 'HEAD'], ['diff', 'HEAD', module_filename]] - process_list = [] - for command in git_commands: - process = subprocess.Popen(['git'] + command, cwd=os.path.split(path)[0], stdout=subprocess.PIPE, - stderr=subprocess.PIPE, startupinfo=startupinfo) - process_list.append(process) - for process, command in zip(process_list, git_commands): - info, err = process.communicate() - hdf5_file[save_path].attrs['git ' + str(command[0])] = info.decode('utf-8') + '\n' + err.decode('utf-8') + if path not in _file_watcher.files: + # Add file to watch list and create its entry in the cache. + _file_watcher.add_file(path) + _file_watcher_callback(path, None, None) + # Get a reference to the current results list in case + # another thread updates _vcs_cache[path] to point at a new + # list during this loop. + results = _vcs_cache[path] + for command, info, err in results: + attribute_str = command[0] + ' ' + command[1] + hdf5_file[save_path].attrs[attribute_str] = (info + '\n' + err) except ImportError: pass except WindowsError if os.name == 'nt' else None: @@ -2554,8 +2627,8 @@ def labscript_cleanup(): compiler.wait_delay = 0 compiler.time_markers = {} compiler._PrimaryBLACS = None - compiler.save_hg_info = LabConfig().getboolean('labscript', 'save_hg_info', fallback=True) - compiler.save_git_info = LabConfig().getboolean('labscript', 'save_git_info', fallback=False) + compiler.save_hg_info = _SAVE_HG_INFO + compiler.save_git_info = _SAVE_GIT_INFO compiler.shot_properties = {} class compiler(object): @@ -2575,8 +2648,8 @@ class compiler(object): wait_delay = 0 time_markers = {} _PrimaryBLACS = None - save_hg_info = LabConfig().getboolean('labscript', 'save_hg_info', fallback=True) - save_git_info = LabConfig().getboolean('labscript', 'save_git_info', fallback=False) + save_hg_info = _SAVE_HG_INFO + save_git_info = _SAVE_GIT_INFO shot_properties = {} # safety measure in case cleanup is called before init From 903cc4397dae90fb4206daa2edcdf4f37a173a17 Mon Sep 17 00:00:00 2001 From: Zak V Date: Wed, 14 Oct 2020 03:59:16 -0400 Subject: [PATCH 09/10] Worked on some thread safety issues with vcs caching. --- labscript/labscript.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/labscript/labscript.py b/labscript/labscript.py index 5337752..fc74f78 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -16,6 +16,7 @@ import sys import subprocess import keyword +import threading from inspect import getcallargs from functools import wraps @@ -2187,8 +2188,10 @@ def generate_connection_table(hdf5_file): # replace the outdated cache entry with a new list of updated vcs commands and # outputs. _vcs_cache = {} +_vcs_cache_rlock = threading.RLock() def _file_watcher_callback(name, info, event): - _vcs_cache[name] = _run_vcs_commands(name) + with _vcs_cache_rlock: + _vcs_cache[name] = _run_vcs_commands(name) _file_watcher = FileWatcher(_file_watcher_callback) @@ -2283,17 +2286,15 @@ def save_labscripts(hdf5_file): # Doesn't seem to want to double count files if you just import the contents of a file within a module continue hdf5_file.create_dataset(save_path, data=open(path).read()) - if path not in _file_watcher.files: - # Add file to watch list and create its entry in the cache. - _file_watcher.add_file(path) - _file_watcher_callback(path, None, None) - # Get a reference to the current results list in case - # another thread updates _vcs_cache[path] to point at a new - # list during this loop. - results = _vcs_cache[path] - for command, info, err in results: - attribute_str = command[0] + ' ' + command[1] - hdf5_file[save_path].attrs[attribute_str] = (info + '\n' + err) + with _vcs_cache_rlock: + if path not in _vcs_cache: + # Add file to watch list and create its entry in the cache. + _file_watcher.add_file(path) + _file_watcher_callback(path, None, None) + # Save the cached vcs output to the file. + for command, info, err in _vcs_cache[path]: + attribute_str = command[0] + ' ' + command[1] + hdf5_file[save_path].attrs[attribute_str] = (info + '\n' + err) except ImportError: pass except WindowsError if os.name == 'nt' else None: From 4716cebf0d5943808f2dd9f8abbf3020ec2cd8bb Mon Sep 17 00:00:00 2001 From: Zak V Date: Thu, 15 Oct 2020 05:47:31 -0400 Subject: [PATCH 10/10] Reduced _vcs_cache_rlock contention in save_labscripts(). --- labscript/labscript.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/labscript/labscript.py b/labscript/labscript.py index fc74f78..887dd66 100644 --- a/labscript/labscript.py +++ b/labscript/labscript.py @@ -2287,10 +2287,12 @@ def save_labscripts(hdf5_file): continue hdf5_file.create_dataset(save_path, data=open(path).read()) with _vcs_cache_rlock: - if path not in _vcs_cache: - # Add file to watch list and create its entry in the cache. - _file_watcher.add_file(path) - _file_watcher_callback(path, None, None) + already_cached = path in _vcs_cache + if not already_cached: + # Add file to watch list and create its entry in the cache. + _file_watcher.add_file(path) + _file_watcher_callback(path, None, None) + with _vcs_cache_rlock: # Save the cached vcs output to the file. for command, info, err in _vcs_cache[path]: attribute_str = command[0] + ' ' + command[1]