Source code for instant.build

"""This module contains the main part of Instant, the build_module function."""

# Copyright (C) 2008-2013 Kent-Andre Mardal
# Copyright (C) 2008-2013 Martin Sandve Alnes
# Copyright (C) 2009-2014 Johan Hake
# Copyright (C) 2009 Ilmar Wilbers
#
# This file is part of Instant.
#
# Instant is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Instant is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Instant. If not, see <http://www.gnu.org/licenses/>.
#
# Alternatively, Instant may be distributed under the terms of the BSD license.

import six
from six import string_types

import io, os, sys, shutil, glob, errno
from itertools import chain

# TODO: Import only the official interface
from .output import *
from .paths import *
from .signatures import *
from .cache import *
from .codegeneration import *
from .locking import file_lock


[docs]def assert_is_str(x): instant_assert(isinstance(x, string_types), "In instant.build_module: Expecting string.")
[docs]def assert_is_bool(x): instant_assert(isinstance(x, bool), "In instant.build_module: Expecting bool.")
[docs]def assert_is_str_list(x): instant_assert(isinstance(x, (list, tuple)), "In instant.build_module: Expecting sequence.") instant_assert(all(isinstance(i, string_types) for i in x), "In instant.build_module: Expecting sequence of strings.")
[docs]def strip_strings(x): assert_is_str_list(x) return [s.strip() for s in x]
[docs]def arg_strings(x): if isinstance(x, string_types): x = x.split() return strip_strings(x)
[docs]def makedirs(path): """ Creates a directory (tree). If directory already excists it does nothing. """ try: os.makedirs(path) except os.error as e: if e.errno != errno.EEXIST: raise
[docs]def copy_files(source, dest, files): """Copy a list of files from a source directory to a destination directory. This may seem a bit complicated, but a lot of this code is error checking. """ if os.path.exists(dest): overwriting = set(files) & set(glob.glob(os.path.join(dest, "*"))) if overwriting: instant_warning("In instant.copy_files: Path '%s' already exists, "\ "overwriting existing files: %r." % (dest, list(overwriting))) else: makedirs(dest) if source != dest: instant_debug("In instant.copy_files: Copying files %r from %r to %r"\ % (files, source, dest)) for f in files: a = os.path.join(source, f) b = os.path.join(dest, f) instant_assert(a != b, "In instant.copy_files: Seems like the "\ "input files are absolute paths, should be relative to "\ "source. (%r, %r)" % (a, b)) instant_assert(os.path.isfile(a), "In instant.copy_files: "\ "Missing source file '%s'." % a) if os.path.isfile(b): os.remove(b) shutil.copyfile(a, b)
[docs]def recompile(modulename, module_path, new_compilation_checksum, build_system="distutils"): """Recompile module if the new checksum is different from the one in the checksum file in the module directory.""" assert(build_system in ["distutils", "cmake"]) # Check if the old checksum matches the new one compilation_checksum_filename = "%s.checksum" % modulename if os.path.exists(compilation_checksum_filename): checksum_file = io.open(compilation_checksum_filename, encoding="utf8") old_compilation_checksum = checksum_file.readline() checksum_file.close() if old_compilation_checksum == new_compilation_checksum: return # Create log file for logging of compilation errors compile_log_filename = os.path.join(module_path, "compile.log") compile_log_filename_dest = os.path.join(get_default_error_dir(), \ modulename, "compile.log") ret = 1 try: compile_log_contents = None instant_info("--- Instant: compiling ---") # TODO: The three blocks below can be made a function and # three calls if build_system == "distutils": # Build extension module with distutils python_interp = sys.executable cmd = python_interp + " setup.py build_ext install --install-platlib=." instant_debug("cmd = %s" % cmd) ret, output = get_status_output(cmd) write_file(compile_log_filename, output) if ret != 0: compile_log_contents = output if os.path.exists(compilation_checksum_filename): os.remove(compilation_checksum_filename) msg = "In instant.recompile: The module did not compile with command '%s', see '%s'" instant_error(msg % (cmd, compile_log_filename_dest)) else: # Build makefile for extension module with cmake cmd = "cmake -DDEBUG=TRUE ."; #cmd = "cmake ."; instant_debug("cmd = %s" % cmd) ret, output = get_status_output(cmd) write_file(compile_log_filename, output) if ret != 0: compile_log_contents = output if os.path.exists(compilation_checksum_filename): os.remove(compilation_checksum_filename) msg = "In instant.recompile: The module did not compile with command '%s', see '%s'" instant_error(msg % (cmd, compile_log_filename_dest)) # Build extension module with cmake generated makefile cmd = "make VERBOSE=1" instant_debug("cmd = %s" % cmd) ret, output = get_status_output(cmd) write_file(compile_log_filename, output, mode="a") if ret != 0: compile_log_contents = output if os.path.exists(compilation_checksum_filename): os.remove(compilation_checksum_filename) msg = "In instant.recompile: The module did not compile with command '%s', see '%s'" instant_error(msg % (cmd, compile_log_filename_dest)) finally: if ret != 0: if "INSTANT_DISPLAY_COMPILE_LOG" in list(os.environ.keys()): instant_warning("") instant_warning("Content of instant compile.log") instant_warning("==============================") instant_warning(compile_log_contents) instant_warning("") # Copy module to error dir module_path = copy_to_cache(module_path, get_default_error_dir(), modulename, check_for_existing_path=False) # Compilation succeeded, write new_compilation_checksum to # checksum_file write_file(compilation_checksum_filename, new_compilation_checksum)
[docs]def copy_to_cache(module_path, cache_dir, modulename, check_for_existing_path=True): "Copy module directory to cache." # Get lock, check if the module exists, _otherwise_ copy the # finished compiled module from /tmp/foo to the cache directory, # and then release lock with file_lock(cache_dir, modulename) as lock: # Validate the path cache_module_path = os.path.join(cache_dir, modulename) if check_for_existing_path and os.path.exists(os.path.join(\ cache_module_path, "finished_copying")): # This indicates a race condition has happened (and is # being avoided!). instant_warning("In instant.build_module: Path '%s' already exists,"\ " but module wasn't found in cache previously. Not overwriting,"\ " assuming this module is valid." % cache_module_path) return cache_module_path # Not deleting anymore, relying on locking system #shutil.rmtree(cache_module_path, ignore_errors=True) # Error checks instant_assert(os.path.isdir(module_path), "In instant.build_module:"\ " Cannot copy non-existing directory %r!" % module_path) if check_for_existing_path and os.path.isdir(cache_module_path): instant_error("In instant.build_module: Cache directory %r shouldn't"\ " exist at this point!" % cache_module_path) instant_debug("In instant.build_module: Copying built module from %r"\ " to cache at %r" % (module_path, cache_module_path)) # Do the copying and mark that we are finished by creating an # empty file finished_copying try: shutil.copytree(module_path, cache_module_path) with io.open(os.path.join(cache_module_path, "finished_copying"), "w", encoding="utf8") as dummy: pass except OSError as e: if e.errno != errno.EEXIST: raise finally: delete_temp_dir() return cache_module_path
[docs]def build_module(modulename=None, source_directory=".", code="", init_code="", additional_definitions="", additional_declarations="", sources=[], wrap_headers=[], local_headers=[], system_headers=[], include_dirs=['.'], library_dirs=[], libraries=[], swigargs=['-c++', '-fcompact', '-O', '-I.', '-small'], swig_include_dirs = [], cppargs=['-O2'], lddargs=[], object_files=[], arrays=[], generate_interface=True, generate_setup=True, cmake_packages=[], signature=None, cache_dir=None): """Generate and compile a module from C/C++ code using SWIG. Arguments: ========== The keyword arguments are as follows: - B{modulename}: - The name you want for the module. If specified, the module will not be cached. If missing, a name will be constructed based on a checksum of the other arguments, and the module will be placed in the global cache. String. - B{source_directory}: - The directory where user supplied files reside. The files given in B{sources}, B{wrap_headers}, and B{local_headers} are expected to exist in this directory. String. - B{code}: - A string containing C or C++ code to be compiled and wrapped. String. - B{init_code}: - Code that should be executed when the Instant module is imported. This code is inserted in the SWIG interface file, and is used for instance for calling C{import_array()} used for the initialization of NumPy arrays. String. - B{additional_definitions}: - Additional definitions (typically needed for inheritance) for interface file. These definitions should be given as triple-quoted strings in the case they span multiple lines, and are placed both in the initial block for C/C++ code (C{%{,%}}-block), and the main section of the interface file. String. - B{additional_declarations}: - Additional declarations (typically needed for inheritance) for interface file. These declarations should be given as triple-quoted strings in the case they span multiple lines, and are plaves in the main section of the interface file. String. - B{sources}: - Source files to compile and link with the module. These files are compiled togehter with the SWIG-generated wrapper file into the final library file. Should reside in directory specified in B{source_directory}. List of strings. - B{wrap_headers}: - Local header files that should be wrapped by SWIG. The files specified will be included both in the initial block for C/C++ code (with a C directive) and in the main section of the interface file (with a SWIG directive). Should reside in directory specified in B{source_directory}. List of strings. - B{local_headers}: - Local header files required to compile the wrapped code. The files specified will be included in the initial block for C/C++ code (with a C directive). Should reside in directory specified in B{source_directory}. List of strings. - B{system_headers}: - System header files required to compile the wrapped code. The files specified will be included in the initial block for C/C++ code (with a C directive). List of strings. - B{include_dirs}: - Directories to search for header files for building the extension module. Needs to be absolute path names. List of strings. - B{library_dirs}: - Directories to search for libraries (C{-l}) for building the extension module. Needs to be absolute paths. List of strings. - B{libraries}: - Libraries needed by the Instant module. The libraries will be linked in from the shared object file. The initial C{-l} is added automatically. List of strings. - B{swigargs}: - List of arguments to swig, e.g. C{["-lpointers.i"]} to include the SWIG pointers.i library. - B{swig_include_dirs}: - A list of directories to include in the 'swig' command. - B{cppargs}: - List of arguments to the compiler, e.g. C{["-Wall", "-fopenmp"]}. - B{lddargs}: - List of arguments to the linker, e.g. C{["-E", "-U"]}. - B{object_files}: - If you want to compile the files yourself. TODO: Not yet supported. - B{arrays}: - A nested list describing the C arrays to be made from NumPy arrays. The SWIG interface for fil NumPy is used. For 1D arrays, the inner list should contain strings with the variable names for the length of the arrays and the array itself. 2D matrices should contain the names of the dimensions in the two directions as well as the name of the array, and 3D tensors should contain the names of the dimensions in the three directions in addition to the name of the array. If the NumPy array har more than four dimensions, the inner list should contain strings with variable names for the number of dimensions, the length in each dimension as a pointer, and the array itself, respectively. - B{generate_interface}: - A bool to indicate if you want to generate the interface files. - B{generate_setup}: - A bool to indicate if you want to generate the setup.py file. - B{cmake_packages}: - A list with CMake configured packages which are used to configure and build the extension module. If used it will override the default behaviour of using distutils. - B{signature}: - A signature string to identify the form instead of the source code. - B{cache_dir}: - A directory to look for cached modules and place new ones. If missing, a default directory is used. Note that the module will not be cached if B{modulename} is specified. The cache directory should not be used for anything else. """ # Store original directory to be able to restore later original_path = os.getcwd() # --- Validate arguments if sys.version_info[0] > 2: swigargs = swigargs + ['-py3'] instant_assert(modulename is None or isinstance(modulename, string_types), "In instant.build_module: Expecting modulename to be string or None.") assert_is_str(source_directory) source_directory = os.path.abspath(source_directory) assert_is_str(code) assert_is_str(init_code) assert_is_str(additional_definitions) assert_is_str(additional_declarations) sources = strip_strings(sources) wrap_headers = strip_strings(wrap_headers) local_headers = strip_strings(local_headers) system_headers = strip_strings(system_headers) include_dirs = strip_strings(include_dirs) library_dirs = strip_strings(library_dirs) libraries = strip_strings(libraries) swigargs = arg_strings(swigargs) swig_include_dirs = strip_strings(swig_include_dirs) cppargs = arg_strings(cppargs) lddargs = arg_strings(lddargs) object_files = strip_strings(object_files) arrays = [strip_strings(a) for a in arrays] assert_is_bool(generate_interface) assert_is_bool(generate_setup) cmake_packages = strip_strings(cmake_packages) instant_assert(signature is None \ or isinstance(signature, string_types) \ or hasattr(signature, "signature"), "In instant.build_module: Expecting modulename to be string or None.") instant_assert(not (signature is not None and modulename is not None), "In instant.build_module: Can't have both modulename and signature.") # --- Replace arguments with defaults if necessary cache_dir = validate_cache_dir(cache_dir) # Split sources by file-suffix (.c or .cpp) csrcs = [f for f in sources if f.endswith('.c') or f.endswith('.C')] cppsrcs = [f for f in sources if f.endswith('.cpp') or f.endswith('.cxx')] instant_assert(len(csrcs) + len(cppsrcs) == len(sources), "In instant.build_module: Source files must have '.c' or '.cpp' suffix") # --- Debugging code instant_debug('In instant.build_module:') instant_debug('::: Begin Arguments :::') instant_debug(' modulename: %r' % modulename) instant_debug(' source_directory: %r' % source_directory) instant_debug(' code: %r' % code) instant_debug(' init_code: %r' % init_code) instant_debug(' additional_definitions: %r' % additional_definitions) instant_debug(' additional_declarations: %r' % additional_declarations) instant_debug(' sources: %r' % sources) instant_debug(' csrcs: %r' % csrcs) instant_debug(' cppsrcs: %r' % cppsrcs) instant_debug(' wrap_headers: %r' % wrap_headers) instant_debug(' local_headers: %r' % local_headers) instant_debug(' system_headers: %r' % system_headers) instant_debug(' include_dirs: %r' % include_dirs) instant_debug(' library_dirs: %r' % library_dirs) instant_debug(' libraries: %r' % libraries) instant_debug(' swigargs: %r' % swigargs) instant_debug(' swig_include_dirs: %r' % swig_include_dirs) instant_debug(' cppargs: %r' % cppargs) instant_debug(' lddargs: %r' % lddargs) instant_debug(' object_files: %r' % object_files) instant_debug(' arrays: %r' % arrays) instant_debug(' generate_interface: %r' % generate_interface) instant_debug(' generate_setup: %r' % generate_setup) instant_debug(' cmake_packages: %r' % cmake_packages) instant_debug(' signature: %r' % signature) instant_debug(' cache_dir: %r' % cache_dir) instant_debug('::: End Arguments :::') # --- Setup module directory, making it and copying # files to it if necessary, and compute a modulename # if it isn't specified explicitly if modulename is None: # Compute a signature if we have none passed by the user: if signature is None: # Collect arguments used for checksum creation, # including everything that affects the interface # file generation and module compilation. checksum_args = ( \ # We don't care about the modulename, that's what # we're trying to construct! #modulename, # We don't care where the user code resides: #source_directory, code, init_code, additional_definitions, additional_declarations, # Skipping filenames, since we use the file contents: #sources, wrap_headers, #local_headers, system_headers, include_dirs, library_dirs, libraries, swig_include_dirs, swigargs, cppargs, lddargs, object_files, arrays, generate_interface, generate_setup, cmake_packages, # The signature isn't defined, and the cache_dir # doesn't affect the module: #signature, cache_dir) sys.version ) allfiles = sources + wrap_headers + local_headers allfiles = [os.path.join(source_directory, f) for f in allfiles] text = "\n".join((str(a) for a in checksum_args)) signature = modulename_from_checksum(compute_checksum(text, allfiles)) modulename = signature moduleids = [signature] else: module, moduleids = check_memory_cache(signature) if module: return module modulename = moduleids[-1] # Look for module in disk cache module = check_disk_cache(modulename, cache_dir, moduleids) if module: return module # Make a temporary module path for compilation module_path = os.path.join(get_temp_dir(), modulename) instant_assert(not os.path.exists(module_path), "In instant.build_module: Not expecting module_path to exist: '%s'"\ % module_path) makedirs(module_path) use_cache = True else: use_cache = False moduleids = [] module_path = os.path.join(original_path, modulename) makedirs(module_path) ## Look for module in memory cache #module, moduleids = check_memory_cache(modulename) #if module: return module #instant_assert(modulename == moduleids[-1] and len(moduleids) == 1, "Logic breach.") ## Look for module in local directory #module = check_disk_cache(modulename, original_path, moduleids) #if module: return module # Wrapping rest of code in try-block to # clean up at the end if something fails. try: # --- Copy user-supplied files to module path module_path = os.path.abspath(module_path) files_to_copy = sources + wrap_headers + local_headers + object_files copy_files(source_directory, module_path, files_to_copy) # At this point, all user input files should reside in module_path # --- Generate additional files in module directory os.chdir(module_path) # Generate __init__.py which imports compiled module contents write_file("__init__.py", "from __future__ import absolute_import\nfrom .%s import *" \ % modulename) # Generate SWIG interface if wanted ifile_name = "%s.i" % modulename if generate_interface: write_interfacefile(ifile_name, modulename, code, init_code, additional_definitions, additional_declarations, system_headers, local_headers, wrap_headers, arrays) # Generate setup.py if wanted if generate_setup and not cmake_packages: setup_name = "setup.py" write_setup(setup_name, modulename, csrcs, cppsrcs, local_headers, include_dirs, library_dirs, libraries, swig_include_dirs, swigargs, cppargs, lddargs) build_system = "distutils" else: write_cmakefile(modulename, cmake_packages, csrcs, cppsrcs, local_headers, include_dirs, library_dirs, libraries, swig_include_dirs, swigargs, cppargs, lddargs) build_system = "cmake" # --- Build module # At this point we have all the files, and can make the total # checksum from all file contents. This is used to decide # whether the module needs recompilation or not. # Compute new_compilation_checksum # Collect arguments used for checksum creation, # including everything that affects the module compilation. # Since the interface file is included in allfiles, # we don't need stuff that modifies it here. checksum_args = ( \ # We don't care about the modulename, that's what # we're trying to construct! #modulename, # We don't care where the user code resides: #source_directory, #code, init_code, #additional_definitions, additional_declarations, # Skipping filenames, since we use the file contents: #sources, wrap_headers, #local_headers, system_headers, include_dirs, library_dirs, libraries, swigargs, swig_include_dirs, cppargs, lddargs, object_files, #arrays, #generate_interface, generate_setup, # The signature isn't defined, and the # cache_dir doesn't affect the module: #signature, cache_dir) ) text = "\n".join((str(a) for a in checksum_args)) allfiles = sources + wrap_headers + local_headers + [ifile_name] new_compilation_checksum = compute_checksum(text, allfiles) # Recompile if necessary recompile(modulename, module_path, new_compilation_checksum, build_system) # --- Load, cache, and return module # Copy compiled module to cache if use_cache: module_path = copy_to_cache(module_path, cache_dir, modulename) # Import module and place in memory cache module = import_and_cache_module(module_path, modulename, moduleids) if not module: instant_error("Failed to import newly compiled module!") instant_debug("In instant.build_module: Returning %s from build_module."\ % module) return module # The end! finally: # Always get back to original directory. os.chdir(original_path) instant_error("In instant.build_module: Should never reach this point!")
# end build_module
[docs]def build_module_vtk(c_code, cache_dir=None): original_path = os.getcwd() cache_dir = validate_cache_dir(cache_dir) signature = modulename_from_checksum(compute_checksum(c_code)) modulename = signature moduleids = [signature] module_path = os.path.join(get_temp_dir(), modulename) makedirs(module_path) os.chdir(module_path) write_cmakefile(modulename) s = generate_interface_file_vtk(signature, c_code) write_vtk_interface_file(signature, c_code) ret, output = get_status_output("cmake -DDEBUG=TRUE . > cmake.log ") ret, output = get_status_output("make > compile.log ") module_path = copy_to_cache(module_path, cache_dir, modulename) os.chdir(original_path) module = import_and_cache_module(module_path, modulename, moduleids) return module
[docs]def build_module_vmtk(c_code, cache_dir=None): original_path = os.getcwd() cache_dir = validate_cache_dir(cache_dir) signature = modulename_from_checksum(compute_checksum(c_code)) modulename = signature moduleids = [signature] module_path = os.path.join(get_temp_dir(), modulename) makedirs(module_path) os.chdir(module_path) write_vmtk_cmakefile(modulename) s = generate_interface_file_vtk(signature, c_code) write_vtk_interface_file(signature, c_code) ret, output = get_status_output("cmake -DDEBUG=TRUE . > cmake.log ") ret, output = get_status_output("make > compile.log ") module_path = copy_to_cache(module_path, cache_dir, modulename) os.chdir(original_path) module = import_and_cache_module(module_path, modulename, moduleids) return module