mirror of https://github.com/tLDP/python-tldp
343 lines
11 KiB
Python
343 lines
11 KiB
Python
#! /usr/bin/python
|
|
# -*- coding: utf8 -*-
|
|
#
|
|
# Copyright (c) 2016 Linux Documentation Project
|
|
|
|
from __future__ import absolute_import, division, print_function
|
|
from __future__ import unicode_literals
|
|
|
|
import os
|
|
import sys
|
|
import stat
|
|
import time
|
|
import errno
|
|
import codecs
|
|
import shutil
|
|
import logging
|
|
import inspect
|
|
from tempfile import NamedTemporaryFile as ntf
|
|
from functools import wraps
|
|
import networkx as nx
|
|
|
|
from tldp.utils import execute, logtimings, writemd5sums
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
preamble = '''#! /bin/bash
|
|
set -x
|
|
set -e
|
|
set -o pipefail
|
|
'''
|
|
|
|
postamble = '''
|
|
# -- end of file'''
|
|
|
|
|
|
def depends(*predecessors):
|
|
'''decorator to be used for constructing build order graph'''
|
|
def anon(f):
|
|
@wraps(f)
|
|
def method(self, *args, **kwargs):
|
|
return f(self, *args, **kwargs)
|
|
method.depends = [x.__name__ for x in predecessors]
|
|
return method
|
|
return anon
|
|
|
|
|
|
class SignatureChecker(object):
|
|
|
|
@classmethod
|
|
def signatureLocation(cls, buf, fname):
|
|
for sig in cls.signatures:
|
|
try:
|
|
sindex = buf.index(sig)
|
|
logger.debug("YES FOUND signature %r in %s at %s; doctype %s.",
|
|
sig, fname, sindex, cls)
|
|
return sindex
|
|
except ValueError:
|
|
logger.debug("not found signature %r in %s for type %s",
|
|
sig, fname, cls.__name__)
|
|
return None
|
|
|
|
|
|
class BaseDoctype(object):
|
|
|
|
def __repr__(self):
|
|
return '<%s:%s>' % (self.__class__.__name__, self.source.stem,)
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
self.source = kwargs.get('source', None)
|
|
self.output = kwargs.get('output', None)
|
|
self.config = kwargs.get('config', None)
|
|
self.removals = set()
|
|
assert self.source is not None
|
|
assert self.output is not None
|
|
assert self.config is not None
|
|
|
|
def cleanup(self):
|
|
stem = self.source.stem
|
|
removals = getattr(self, 'removals', None)
|
|
if removals:
|
|
for fn in removals:
|
|
logger.debug("%s cleaning up intermediate file %s", stem, fn)
|
|
try:
|
|
os.unlink(fn)
|
|
except OSError as e:
|
|
if e.errno is errno.ENOENT:
|
|
logger.error("%s missing file at cleanup %s", stem, fn)
|
|
else:
|
|
raise e
|
|
|
|
def build_precheck(self):
|
|
classname = self.__class__.__name__
|
|
if self.config.script:
|
|
return True
|
|
for tool, validator in self.required.items():
|
|
thing = getattr(self.config, tool, None)
|
|
logger.debug("%s, tool = %s, thing = %s", classname, tool, thing)
|
|
if thing is None:
|
|
logger.error("%s missing required tool %s, skipping...",
|
|
classname, tool)
|
|
return False
|
|
assert validator(thing)
|
|
return True
|
|
|
|
def clear_output(self, **kwargs):
|
|
'''remove the entire output directory
|
|
|
|
This method must be --script aware. The method execute_shellscript()
|
|
generates scripts into the directory that would be removed. Thus, the
|
|
behaviour is different depending on --script mode or --build mode.
|
|
'''
|
|
logger.debug("%s removing dir %s.",
|
|
self.output.stem, self.output.dirname)
|
|
if self.config.script:
|
|
s = 'test -d "{output.dirname}" && rm -rf -- "{output.dirname}"'
|
|
return self.shellscript(s, **kwargs)
|
|
if os.path.exists(self.output.dirname):
|
|
shutil.rmtree(self.output.dirname)
|
|
return True
|
|
|
|
def mkdir_output(self, **kwargs):
|
|
'''create a new output directory
|
|
|
|
This method must be --script aware. The method execute_shellscript()
|
|
generates scripts into the directory that would be removed. Thus, the
|
|
behaviour is different depending on --script mode or --build mode.
|
|
'''
|
|
logger.debug("%s creating dir %s.",
|
|
self.output.stem, self.output.dirname)
|
|
if self.config.script:
|
|
s = 'mkdir -p -- "{output.logdir}"'
|
|
return self.shellscript(s, **kwargs)
|
|
for d in (self.output.dirname, self.output.logdir):
|
|
if not os.path.isdir(d):
|
|
os.mkdir(d)
|
|
return True
|
|
|
|
def chdir_output(self, **kwargs):
|
|
'''chdir to the output directory (or write the script that would)'''
|
|
logger.debug("%s chdir to dir %s.",
|
|
self.output.stem, self.output.dirname)
|
|
if self.config.script:
|
|
s = '''
|
|
# - - - - - {source.stem} - - - - - -
|
|
|
|
cd -- "{output.dirname}"'''
|
|
return self.shellscript(s, **kwargs)
|
|
os.chdir(self.output.dirname)
|
|
return True
|
|
|
|
def generate_md5sums(self, **kwargs):
|
|
logger.debug("%s generating MD5SUMS in %s.",
|
|
self.output.stem, self.output.dirname)
|
|
timestr = time.strftime('%F-%T', time.gmtime())
|
|
md5file = self.output.MD5SUMS
|
|
if self.config.script:
|
|
l = list()
|
|
for fname, hashval in sorted(self.source.md5sums.items()):
|
|
l.append('# {} {}'.format(hashval, fname))
|
|
md5s = '\n'.join(l)
|
|
s = '''# -- MD5SUMS file from source tree at {}
|
|
#
|
|
# md5sum > {} -- {}
|
|
#
|
|
{}
|
|
#'''
|
|
s = s.format(timestr,
|
|
md5file,
|
|
' '.join(self.source.md5sums.keys()),
|
|
md5s)
|
|
return self.shellscript(s, **kwargs)
|
|
header = '# -- MD5SUMS for {}'.format(self.source.stem)
|
|
writemd5sums(md5file, self.source.md5sums, header=header)
|
|
return True
|
|
|
|
def copy_static_resources(self, **kwargs):
|
|
logger.debug("%s copy resources %s.",
|
|
self.output.stem, self.output.dirname)
|
|
source = list()
|
|
for d in self.config.resources:
|
|
fullpath = os.path.join(self.source.dirname, d)
|
|
fullpath = os.path.abspath(fullpath)
|
|
if os.path.isdir(fullpath):
|
|
source.append('"' + fullpath + '"')
|
|
if not source:
|
|
logger.debug("%s no images or resources to copy", self.source.stem)
|
|
return True
|
|
s = 'rsync --archive --verbose %s ./' % (' '.join(source))
|
|
return self.shellscript(s, **kwargs)
|
|
|
|
def hook_build_success(self):
|
|
stem = self.output.stem
|
|
logdir = self.output.logdir
|
|
dirname = self.output.dirname
|
|
logger.info("%s build SUCCESS %s.", stem, dirname)
|
|
logger.debug("%s removing logs %s)", stem, logdir)
|
|
if os.path.isdir(logdir):
|
|
shutil.rmtree(logdir)
|
|
return True
|
|
|
|
def hook_build_failure(self):
|
|
pass
|
|
|
|
def shellscript(self, script, **kwargs):
|
|
if self.config.build:
|
|
return self.execute_shellscript(script, **kwargs)
|
|
elif self.config.script:
|
|
return self.dump_shellscript(script, **kwargs)
|
|
else:
|
|
etext = '%s in shellscript, neither --build nor --script'
|
|
raise Exception(etext % (self.source.stem,))
|
|
|
|
@logtimings(logger.debug)
|
|
def dump_shellscript(self, script, preamble=preamble,
|
|
postamble=postamble, **kwargs):
|
|
source = self.source
|
|
output = self.output
|
|
config = self.config
|
|
file = kwargs.get('file', sys.stdout)
|
|
s = script.format(output=output, source=source, config=config)
|
|
print('', file=file)
|
|
print(s, file=file)
|
|
return True
|
|
|
|
@logtimings(logger.debug)
|
|
def execute_shellscript(self, script, preamble=preamble,
|
|
postamble=postamble, **kwargs):
|
|
source = self.source
|
|
output = self.output
|
|
config = self.config
|
|
|
|
logdir = output.logdir
|
|
prefix = source.doctype.__name__ + '-'
|
|
|
|
s = script.format(output=output, source=source, config=config)
|
|
tf = ntf(dir=logdir, prefix=prefix, suffix='.sh', delete=False)
|
|
tf.close()
|
|
with codecs.open(tf.name, 'w', encoding='utf-8') as f:
|
|
if preamble:
|
|
f.write(preamble)
|
|
f.write(s)
|
|
if postamble:
|
|
f.write(postamble)
|
|
|
|
mode = stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR
|
|
os.chmod(tf.name, mode)
|
|
|
|
cmd = [tf.name]
|
|
result = execute(cmd, logdir=logdir)
|
|
if result != 0:
|
|
with codecs.open(tf.name, encoding='utf-8') as f:
|
|
for line in f:
|
|
logger.info("Script: %s", line.rstrip())
|
|
return False
|
|
return True
|
|
|
|
def build_prepare(self, **kwargs):
|
|
stem = self.source.stem
|
|
classname = self.__class__.__name__
|
|
order = ['build_precheck',
|
|
'clear_output',
|
|
'mkdir_output',
|
|
'chdir_output',
|
|
'generate_md5sums',
|
|
'copy_static_resources',
|
|
]
|
|
|
|
for methname in order:
|
|
method = getattr(self, methname, None)
|
|
assert method is not None
|
|
logger.info("%s calling method %s.%s",
|
|
stem, classname, method.__name__)
|
|
if not method(**kwargs):
|
|
logger.error("%s called method %s.%s failed, skipping...",
|
|
stem, classname, method.__name__)
|
|
return False
|
|
return True
|
|
|
|
def determinebuildorder(self):
|
|
graph = nx.DiGraph()
|
|
d = dict(inspect.getmembers(self, inspect.ismethod))
|
|
for name, member in d.items():
|
|
predecessors = getattr(member, 'depends', None)
|
|
if predecessors:
|
|
for pred in predecessors:
|
|
method = d.get(pred, None)
|
|
assert method is not None
|
|
graph.add_edge(method, member)
|
|
order = nx.dag.topological_sort(graph)
|
|
return order
|
|
|
|
@logtimings(logger.debug)
|
|
def build_fullrun(self, **kwargs):
|
|
stem = self.source.stem
|
|
order = self.determinebuildorder()
|
|
logger.debug("%s build order %r", self.source.stem, order)
|
|
for method in order:
|
|
classname = self.__class__.__name__
|
|
logger.info("%s calling method %s.%s",
|
|
stem, classname, method.__name__)
|
|
if not method(**kwargs):
|
|
logger.error("%s called method %s.%s failed, skipping...",
|
|
stem, classname, method.__name__)
|
|
return False
|
|
return True
|
|
|
|
@logtimings(logger.info)
|
|
def generate(self, **kwargs):
|
|
# -- perform build preparation steps;
|
|
# - check for all executables and data files
|
|
# - clear output dir
|
|
# - make output dir
|
|
# - chdir to output dir
|
|
# - copy source images/resources to output dir
|
|
#
|
|
if not self.config.script:
|
|
opwd = os.getcwd()
|
|
if not self.build_prepare():
|
|
return False
|
|
|
|
# -- build
|
|
#
|
|
result = self.build_fullrun(**kwargs)
|
|
|
|
# -- always clean the kitchen
|
|
#
|
|
self.cleanup()
|
|
|
|
# -- report on result and/or cleanup
|
|
#
|
|
if result:
|
|
self.hook_build_success()
|
|
else:
|
|
self.hook_build_failure()
|
|
|
|
if not self.config.script:
|
|
os.chdir(opwd)
|
|
|
|
return result
|
|
|
|
#
|
|
# -- end of file
|