2016-02-11 03:22:23 +00:00
|
|
|
#! /usr/bin/python
|
2016-02-18 21:25:02 +00:00
|
|
|
# -*- coding: utf8 -*-
|
2016-02-11 03:22:23 +00:00
|
|
|
|
2016-02-11 19:28:38 +00:00
|
|
|
from __future__ import absolute_import, division, print_function
|
2016-02-11 03:22:23 +00:00
|
|
|
|
2016-02-15 21:58:32 +00:00
|
|
|
import os
|
2016-02-24 05:58:03 +00:00
|
|
|
import stat
|
2016-02-26 19:58:43 +00:00
|
|
|
import errno
|
2016-02-26 08:30:37 +00:00
|
|
|
import logging
|
2016-02-24 05:58:03 +00:00
|
|
|
from tempfile import NamedTemporaryFile as ntf
|
2016-02-26 08:30:37 +00:00
|
|
|
from functools import wraps
|
|
|
|
import networkx as nx
|
|
|
|
|
2016-02-27 07:02:00 +00:00
|
|
|
from tldp.utils import execute, logtimings
|
2016-02-24 05:58:03 +00:00
|
|
|
|
2016-02-25 20:29:55 +00:00
|
|
|
logger = logging.getLogger(__name__)
|
2016-02-25 19:37:14 +00:00
|
|
|
|
2016-02-26 08:30:37 +00:00
|
|
|
preamble = '''#! /bin/bash
|
|
|
|
set -x
|
|
|
|
set -e
|
|
|
|
set -o pipefail
|
|
|
|
|
|
|
|
'''
|
|
|
|
|
2016-02-26 18:10:45 +00:00
|
|
|
postamble = '''
|
|
|
|
|
|
|
|
'''
|
|
|
|
|
2016-02-26 19:58:43 +00:00
|
|
|
|
2016-02-26 08:30:37 +00:00
|
|
|
def depends(graph, *predecessors):
|
|
|
|
'''decorator to be used for constructing build order graph'''
|
|
|
|
def anon(f):
|
|
|
|
for dep in predecessors:
|
|
|
|
graph.add_edge(dep.__name__, f.__name__)
|
|
|
|
|
|
|
|
@wraps(f)
|
|
|
|
def method(self, *args, **kwargs):
|
|
|
|
return f(self, *args, **kwargs)
|
|
|
|
return method
|
|
|
|
return anon
|
2016-02-11 03:22:23 +00:00
|
|
|
|
|
|
|
|
|
|
|
class SignatureChecker(object):
|
|
|
|
|
|
|
|
@classmethod
|
2016-02-12 20:25:34 +00:00
|
|
|
def signatureLocation(cls, f):
|
|
|
|
f.seek(0)
|
2016-02-12 21:24:55 +00:00
|
|
|
buf = f.read(1024).lower()
|
2016-02-11 03:22:23 +00:00
|
|
|
for sig in cls.signatures:
|
|
|
|
try:
|
2016-02-12 21:24:55 +00:00
|
|
|
sindex = buf.index(sig.lower())
|
2016-03-02 03:51:19 +00:00
|
|
|
logger.debug("Found signature %r in %s at %s; doctype %s.",
|
2016-02-12 20:25:34 +00:00
|
|
|
sig, f.name, sindex, cls)
|
2016-02-11 03:22:23 +00:00
|
|
|
return sindex
|
|
|
|
except ValueError:
|
2016-03-02 03:51:19 +00:00
|
|
|
logger.debug("Signature %r not found in %s for type %s",
|
2016-02-12 20:25:34 +00:00
|
|
|
sig, f.name, cls.__name__)
|
2016-02-11 03:22:23 +00:00
|
|
|
return None
|
|
|
|
|
2016-02-15 21:58:32 +00:00
|
|
|
|
|
|
|
class BaseDoctype(object):
|
|
|
|
|
2016-02-27 07:18:50 +00:00
|
|
|
def __repr__(self):
|
|
|
|
return '<%s:%s>' % (self.__class__.__name__, self.source.stem,)
|
|
|
|
|
2016-02-15 21:58:32 +00:00
|
|
|
def __init__(self, *args, **kwargs):
|
2016-02-16 07:58:50 +00:00
|
|
|
self.source = kwargs.get('source', None)
|
|
|
|
self.output = kwargs.get('output', None)
|
2016-02-23 17:43:27 +00:00
|
|
|
self.config = kwargs.get('config', None)
|
2016-02-26 19:58:43 +00:00
|
|
|
self.removals = list()
|
2016-03-01 16:20:57 +00:00
|
|
|
assert self.source is not None
|
|
|
|
assert self.output is not None
|
|
|
|
assert self.config is not None
|
2016-02-15 21:58:32 +00:00
|
|
|
|
2016-02-26 19:58:43 +00:00
|
|
|
def cleanup(self):
|
|
|
|
stem = self.source.stem
|
|
|
|
removals = getattr(self, 'removals', None)
|
|
|
|
if removals:
|
|
|
|
for fn in removals:
|
2016-02-26 20:06:34 +00:00
|
|
|
logger.debug("%s cleaning up intermediate file %s", stem, fn)
|
2016-02-26 19:58:43 +00:00
|
|
|
try:
|
|
|
|
os.unlink(fn)
|
|
|
|
except OSError as e:
|
|
|
|
if e.errno is errno.ENOENT:
|
|
|
|
logger.error("%s missing file at cleanup %s", stem, fn)
|
|
|
|
else:
|
|
|
|
raise e
|
|
|
|
|
2016-02-23 20:18:17 +00:00
|
|
|
def build_precheck(self):
|
2016-02-26 19:19:30 +00:00
|
|
|
classname = self.__class__.__name__
|
2016-02-23 20:18:17 +00:00
|
|
|
for tool, validator in self.required.items():
|
|
|
|
thing = getattr(self.config, tool, None)
|
2016-02-26 19:19:30 +00:00
|
|
|
if thing is None:
|
2016-02-26 19:58:43 +00:00
|
|
|
logger.error("%s missing required tool %s, skipping...",
|
2016-02-26 19:19:30 +00:00
|
|
|
classname, tool)
|
|
|
|
return False
|
2016-02-23 20:18:17 +00:00
|
|
|
assert validator(thing)
|
|
|
|
return True
|
|
|
|
|
2016-02-26 20:06:34 +00:00
|
|
|
def hook_build_success(self):
|
|
|
|
self.cleanup()
|
|
|
|
|
|
|
|
def hook_build_failure(self):
|
|
|
|
self.cleanup()
|
|
|
|
|
2016-02-27 07:02:00 +00:00
|
|
|
@logtimings(logger.debug)
|
2016-02-26 18:10:45 +00:00
|
|
|
def shellscript(self, script, preamble=preamble, postamble=postamble):
|
2016-02-24 05:58:03 +00:00
|
|
|
source = self.source
|
|
|
|
output = self.output
|
|
|
|
config = self.config
|
|
|
|
|
|
|
|
logdir = output.logdir
|
|
|
|
prefix = source.doctype.__name__ + '-'
|
|
|
|
|
|
|
|
s = script.format(output=output, source=source, config=config)
|
|
|
|
tf = ntf(dir=logdir, prefix=prefix, suffix='.sh', delete=False)
|
2016-02-26 08:30:37 +00:00
|
|
|
if preamble:
|
|
|
|
tf.write(preamble)
|
2016-02-24 05:58:03 +00:00
|
|
|
tf.write(s)
|
2016-02-26 18:10:45 +00:00
|
|
|
if postamble:
|
|
|
|
tf.write(postamble)
|
2016-02-24 05:58:03 +00:00
|
|
|
tf.close()
|
|
|
|
|
|
|
|
mode = stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR
|
|
|
|
os.chmod(tf.name, mode)
|
|
|
|
|
|
|
|
cmd = [tf.name]
|
|
|
|
result = execute(cmd, logdir=logdir)
|
|
|
|
if result != 0:
|
2016-02-26 18:10:45 +00:00
|
|
|
with open(tf.name) as f:
|
|
|
|
for line in f:
|
|
|
|
logger.debug("Script: %s", line.rstrip())
|
2016-02-24 05:58:03 +00:00
|
|
|
return False
|
2016-02-26 08:30:37 +00:00
|
|
|
return True
|
|
|
|
|
2016-02-27 07:02:00 +00:00
|
|
|
@logtimings(logger.debug)
|
2016-02-26 08:30:37 +00:00
|
|
|
def buildall(self):
|
2016-02-26 09:00:43 +00:00
|
|
|
stem = self.source.stem
|
2016-02-26 08:30:37 +00:00
|
|
|
order = nx.dag.topological_sort(self.graph)
|
|
|
|
logger.debug("%s build order %r", self.source.stem, order)
|
|
|
|
for dep in order:
|
|
|
|
method = getattr(self, dep, None)
|
|
|
|
assert method is not None
|
2016-02-26 19:19:30 +00:00
|
|
|
classname = self.__class__.__name__
|
|
|
|
logger.info("%s calling method %s.%s", stem, classname, dep)
|
2016-02-26 08:30:37 +00:00
|
|
|
if not method():
|
2016-02-26 09:00:43 +00:00
|
|
|
logger.error("%s reported method %s failure, skipping...",
|
|
|
|
stem, dep)
|
2016-02-26 08:30:37 +00:00
|
|
|
return False
|
2016-02-24 06:37:59 +00:00
|
|
|
return True
|
2016-02-24 05:58:03 +00:00
|
|
|
|
2016-02-27 07:02:00 +00:00
|
|
|
@logtimings(logger.info)
|
2016-02-15 21:58:32 +00:00
|
|
|
def generate(self):
|
2016-02-26 19:19:30 +00:00
|
|
|
stem = self.source.stem
|
|
|
|
classname = self.__class__.__name__
|
|
|
|
|
2016-02-23 19:56:58 +00:00
|
|
|
# -- the output directory gets to prepare; must return True
|
|
|
|
#
|
|
|
|
# -- the processor gets to prepare; must return True
|
|
|
|
#
|
|
|
|
if not self.build_precheck():
|
2016-02-24 05:58:03 +00:00
|
|
|
logger.warning("%s %s failed (%s), skipping to next build",
|
2016-02-27 19:13:58 +00:00
|
|
|
stem, 'build_precheck', classname)
|
2016-02-23 19:56:58 +00:00
|
|
|
return False
|
|
|
|
|
2016-02-27 19:13:58 +00:00
|
|
|
if not self.output.hook_prebuild():
|
|
|
|
logger.warning("%s %s failed (%s), skipping to next build",
|
|
|
|
stem, 'hook_prebuild', classname)
|
|
|
|
return False
|
|
|
|
|
|
|
|
opwd = os.getcwd()
|
|
|
|
os.chdir(self.output.dirname)
|
|
|
|
|
2016-02-26 20:06:34 +00:00
|
|
|
# -- now, we can try to build everything; this is the BIG WORK!
|
2016-02-23 19:56:58 +00:00
|
|
|
#
|
2016-02-26 20:06:34 +00:00
|
|
|
result = self.buildall()
|
2016-02-26 19:19:30 +00:00
|
|
|
|
2016-02-18 02:31:51 +00:00
|
|
|
if result:
|
2016-02-26 19:19:30 +00:00
|
|
|
self.hook_build_success() # -- processor
|
|
|
|
self.output.hook_build_success() # -- output document
|
2016-02-18 02:31:51 +00:00
|
|
|
else:
|
2016-02-26 19:19:30 +00:00
|
|
|
self.hook_build_failure() # -- processor
|
|
|
|
self.output.hook_build_failure() # -- output document
|
|
|
|
|
2016-02-23 19:56:58 +00:00
|
|
|
os.chdir(opwd)
|
2016-02-26 19:19:30 +00:00
|
|
|
|
2016-02-23 19:08:04 +00:00
|
|
|
return result
|
2016-02-15 21:58:32 +00:00
|
|
|
|
2016-02-11 03:22:23 +00:00
|
|
|
#
|
|
|
|
# -- end of file
|