Source code for magni.utils.multiprocessing._util

    Copyright (c) 2014-2017, Magni developers.
    All rights reserved.
    See LICENSE.rst for further information.

Module providing the public class of the magni.utils.multiprocessing


from __future__ import division
import multiprocessing as mp

import tables

from magni.utils.validation import decorate_validation as _decorate_validation

_lock = mp.Lock()

[docs]class File(): """ Control pytables access to hdf5 files when using multiprocessing. `File` retains the interface of `tables.open_file` and should only be used in 'with' statements (see Examples). Parameters ---------- args : tuple The arguments that are passed to 'tables.open_file'. kwargs : dict The keyword arguments that are passed to 'tables.open_file'. See Also -------- tables.open_file : The wrapped function. Notes ----- Internally the module uses a global lock which is shared amongst all files. This solution is simple and does not entail significant overhead. However, the wait time introduced when using multiple files at the same time can be significant. Examples -------- The class is used in the following way: >>> from magni.utils.multiprocessing._util import File >>> with File('database.hdf5', 'a') as f: ... pass # execute something involving the opened file """ def __init__(self, *args, **kwargs): @_decorate_validation def validate_input(): if len(args) == 0 and 'filename' not in kwargs: raise ValueError('File must be called with a filename ' 'argument.') validate_input() self._args = args self._kwargs = kwargs if len(args) > 0: self._filename = args[0] else: self._filename = kwargs['filename']
[docs] def __enter__(self): """ Acquire the global lock before opening and returning the file. Returns ------- file : tables.File The file specified in the call to `__init__`. """ _lock.acquire() self._file = tables.open_file(*self._args, **self._kwargs) return self._file
[docs] def __exit__(self, type, value, traceback): """ Release the global lock after closing the file. Parameters ---------- type : type The type of the exception raised, if any. value : Exception The exception rasied, if any. traceback : traceback The traceback of the exception raised, if any. """ self._file.close() _lock.release()