python-tldp/tldp/doctypes/common.py

#! /usr/bin/python
# -*- coding: utf8 -*-
#
# Copyright (c) 2016 Linux Documentation Project

from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals

import os
import sys
import stat
import time
import errno
import codecs
import shutil
import logging
import inspect
from tempfile import NamedTemporaryFile as ntf
from functools import wraps
import networkx as nx

from tldp.utils import execute, logtimings, writemd5sums

logger = logging.getLogger(__name__)

preamble = '''#! /bin/bash
set -x
set -e
set -o pipefail
'''

postamble = '''
# -- end of file'''


def depends(*predecessors):
    '''decorator to be used for constructing build order graph'''
    def anon(f):
        @wraps(f)
        def method(self, *args, **kwargs):
            return f(self, *args, **kwargs)
        method.depends = [x.__name__ for x in predecessors]
        return method
    return anon


class SignatureChecker(object):

    @classmethod
    def signatureLocation(cls, buf, fname):
        for sig in cls.signatures:
            try:
                sindex = buf.index(sig)
                logger.debug("YES FOUND signature %r in %s at %s; doctype %s.",
                             sig, fname, sindex, cls)
                return sindex
            except ValueError:
                logger.debug("not found signature %r in %s for type %s",
                             sig, fname, cls.__name__)
        return None


class BaseDoctype(object):

    def __repr__(self):
        return '<%s:%s>' % (self.__class__.__name__, self.source.stem,)

    def __init__(self, *args, **kwargs):
        self.source = kwargs.get('source', None)
        self.output = kwargs.get('output', None)
        self.config = kwargs.get('config', None)
        self.removals = set()
        assert self.source is not None
        assert self.output is not None
        assert self.config is not None

    def cleanup(self):
        stem = self.source.stem
        removals = getattr(self, 'removals', None)
        if removals:
            for fn in removals:
                logger.debug("%s cleaning up intermediate file %s", stem, fn)
                try:
                    os.unlink(fn)
                except OSError as e:
                    if e.errno is errno.ENOENT:
                        logger.error("%s missing file at cleanup %s", stem, fn)
                    else:
                        raise e

    def build_precheck(self):
        classname = self.__class__.__name__
        if self.config.script:
            return True
        for tool, validator in self.required.items():
            thing = getattr(self.config, tool, None)
            logger.debug("%s, tool = %s, thing = %s", classname, tool, thing)
            if thing is None:
                logger.error("%s missing required tool %s, skipping...",
                             classname, tool)
                return False
            assert validator(thing)
        return True

    def clear_output(self, **kwargs):
        '''remove the entire output directory

        This method must be --script aware.  The method execute_shellscript()
        generates scripts into the directory that would be removed.  Thus, the
        behaviour is different depending on --script mode or --build mode.
        '''
        logger.debug("%s removing dir   %s.",
                     self.output.stem, self.output.dirname)
        if self.config.script:
            s = 'test -d "{output.dirname}" && rm -rf -- "{output.dirname}"'
            return self.shellscript(s, **kwargs)
        if os.path.exists(self.output.dirname):
            shutil.rmtree(self.output.dirname)
        return True

    def mkdir_output(self, **kwargs):
        '''create a new output directory

        This method must be --script aware.  The method execute_shellscript()
        generates scripts into the directory that would be removed.  Thus, the
        behaviour is different depending on --script mode or --build mode.
        '''
        logger.debug("%s creating dir   %s.",
                     self.output.stem, self.output.dirname)
        if self.config.script:
            s = 'mkdir -p -- "{output.logdir}"'
            return self.shellscript(s, **kwargs)
        for d in (self.output.dirname, self.output.logdir):
            if not os.path.isdir(d):
                os.mkdir(d)
        return True

    def chdir_output(self, **kwargs):
        '''chdir to the output directory (or write the script that would)'''
        logger.debug("%s chdir to dir   %s.",
                     self.output.stem, self.output.dirname)
        if self.config.script:
            s = '''
# - - - - - {source.stem} - - - - - -

cd -- "{output.dirname}"'''
            return self.shellscript(s, **kwargs)
        os.chdir(self.output.dirname)
        return True

    def generate_md5sums(self, **kwargs):
        logger.debug("%s generating MD5SUMS in %s.",
                     self.output.stem, self.output.dirname)
        timestr = time.strftime('%F-%T', time.gmtime())
        md5file = self.output.MD5SUMS
        if self.config.script:
            l = list()
            for fname, hashval in sorted(self.source.md5sums.items()):
                l.append('# {}  {}'.format(hashval, fname))
            md5s = '\n'.join(l)
            s = '''# -- MD5SUMS file from source tree at {}
#
# md5sum > {} -- {}
#
{}
#'''
            s = s.format(timestr,
                         md5file,
                         ' '.join(self.source.md5sums.keys()),
                         md5s)
            return self.shellscript(s, **kwargs)
        header = '# -- MD5SUMS for {}'.format(self.source.stem)
        writemd5sums(md5file, self.source.md5sums, header=header)
        return True

    def copy_static_resources(self, **kwargs):
        logger.debug("%s copy resources %s.",
                     self.output.stem, self.output.dirname)
        source = list()
        for d in self.config.resources:
            fullpath = os.path.join(self.source.dirname, d)
            fullpath = os.path.abspath(fullpath)
            if os.path.isdir(fullpath):
                source.append('"' + fullpath + '"')
        if not source:
            logger.debug("%s no images or resources to copy", self.source.stem)
            return True
        s = 'rsync --archive --verbose %s ./' % (' '.join(source))
        return self.shellscript(s, **kwargs)

    def hook_build_success(self):
        stem = self.output.stem
        logdir = self.output.logdir
        dirname = self.output.dirname
        logger.info("%s build SUCCESS  %s.", stem, dirname)
        logger.debug("%s removing logs  %s)", stem, logdir)
        if os.path.isdir(logdir):
            shutil.rmtree(logdir)
        return True

    def hook_build_failure(self):
        pass

    def shellscript(self, script, **kwargs):
        if self.config.build:
            return self.execute_shellscript(script, **kwargs)
        elif self.config.script:
            return self.dump_shellscript(script, **kwargs)
        else:
            etext = '%s in shellscript, neither --build nor --script'
            raise Exception(etext % (self.source.stem,))

    @logtimings(logger.debug)
    def dump_shellscript(self, script, preamble=preamble,
                         postamble=postamble, **kwargs):
        source = self.source
        output = self.output
        config = self.config
        file = kwargs.get('file', sys.stdout)
        s = script.format(output=output, source=source, config=config)
        print('', file=file)
        print(s, file=file)
        return True

    @logtimings(logger.debug)
    def execute_shellscript(self, script, preamble=preamble,
                            postamble=postamble, **kwargs):
        source = self.source
        output = self.output
        config = self.config

        logdir = output.logdir
        prefix = source.doctype.__name__ + '-'

        s = script.format(output=output, source=source, config=config)
        tf = ntf(dir=logdir, prefix=prefix, suffix='.sh', delete=False)
        tf.close()
        with codecs.open(tf.name, 'w', encoding='utf-8') as f:
            if preamble:
                f.write(preamble)
            f.write(s)
            if postamble:
                f.write(postamble)

        mode = stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR
        os.chmod(tf.name, mode)

        cmd = [tf.name]
        result = execute(cmd, logdir=logdir)
        if result != 0:
            with codecs.open(tf.name, encoding='utf-8') as f:
                for line in f:
                    logger.info("Script: %s", line.rstrip())
            return False
        return True

    def build_prepare(self, **kwargs):
        stem = self.source.stem
        classname = self.__class__.__name__
        order = ['build_precheck',
                 'clear_output',
                 'mkdir_output',
                 'chdir_output',
                 'generate_md5sums',
                 'copy_static_resources',
                 ]

        for methname in order:
            method = getattr(self, methname, None)
            assert method is not None
            logger.info("%s calling method %s.%s",
                        stem, classname, method.__name__)
            if not method(**kwargs):
                logger.error("%s called method  %s.%s failed, skipping...",
                             stem, classname, method.__name__)
                return False
        return True

    def determinebuildorder(self):
        graph = nx.DiGraph()
        d = dict(inspect.getmembers(self, inspect.ismethod))
        for name, member in d.items():
            predecessors = getattr(member, 'depends', None)
            if predecessors:
                for pred in predecessors:
                    method = d.get(pred, None)
                    assert method is not None
                    graph.add_edge(method, member)
        order = nx.dag.topological_sort(graph)
        return order

    @logtimings(logger.debug)
    def build_fullrun(self, **kwargs):
        stem = self.source.stem
        order = self.determinebuildorder()
        logger.debug("%s build order %r", self.source.stem, order)
        for method in order:
            classname = self.__class__.__name__
            logger.info("%s calling method %s.%s",
                        stem, classname, method.__name__)
            if not method(**kwargs):
                logger.error("%s called method  %s.%s failed, skipping...",
                             stem, classname, method.__name__)
                return False
        return True

    @logtimings(logger.info)
    def generate(self, **kwargs):
        # -- perform build preparation steps;
        #     - check for all executables and data files
        #     - clear output dir
        #     - make output dir
        #     - chdir to output dir
        #     - copy source images/resources to output dir
        #
        if not self.config.script:
            opwd = os.getcwd()
        if not self.build_prepare():
            return False

        # -- build
        #
        result = self.build_fullrun(**kwargs)

        # -- always clean the kitchen
        #
        self.cleanup()

        # -- report on result and/or cleanup
        #
        if result:
            self.hook_build_success()
        else:
            self.hook_build_failure()

        if not self.config.script:
            os.chdir(opwd)

        return result

#
# -- end of file