2016-02-11 03:22:23 +00:00
|
|
|
#! /usr/bin/python
|
2016-02-18 21:25:02 +00:00
|
|
|
# -*- coding: utf8 -*-
|
2016-02-11 03:22:23 +00:00
|
|
|
|
2016-02-11 19:28:38 +00:00
|
|
|
from __future__ import absolute_import, division, print_function
|
2016-03-15 05:18:09 +00:00
|
|
|
from __future__ import unicode_literals
|
2016-02-11 19:28:38 +00:00
|
|
|
|
2016-02-12 18:31:50 +00:00
|
|
|
import os
|
2016-02-27 07:02:00 +00:00
|
|
|
import time
|
2016-02-17 03:52:07 +00:00
|
|
|
import errno
|
2016-03-15 04:42:07 +00:00
|
|
|
import codecs
|
2016-04-02 17:45:53 +00:00
|
|
|
import hashlib
|
2016-02-17 07:42:22 +00:00
|
|
|
import operator
|
2016-02-15 22:03:56 +00:00
|
|
|
import subprocess
|
2016-02-17 07:42:22 +00:00
|
|
|
import functools
|
2016-02-27 07:02:00 +00:00
|
|
|
from functools import wraps
|
2016-03-07 23:04:16 +00:00
|
|
|
from tempfile import mkstemp, mkdtemp
|
2016-02-11 03:22:23 +00:00
|
|
|
import logging
|
2016-02-25 20:29:12 +00:00
|
|
|
logger = logging.getLogger(__name__)
|
2016-02-11 03:22:23 +00:00
|
|
|
|
2016-03-07 23:04:16 +00:00
|
|
|
opa = os.path.abspath
|
|
|
|
opb = os.path.basename
|
|
|
|
opd = os.path.dirname
|
|
|
|
opj = os.path.join
|
|
|
|
|
2016-02-18 02:16:20 +00:00
|
|
|
logdir = 'tldp-document-build-logs'
|
2016-02-11 03:22:23 +00:00
|
|
|
|
2016-02-18 03:38:27 +00:00
|
|
|
|
2016-02-27 07:02:00 +00:00
|
|
|
def logtimings(logmethod):
|
|
|
|
def anon(f):
|
|
|
|
@wraps(f)
|
2016-02-27 07:18:24 +00:00
|
|
|
def timing(*args, **kwargs):
|
2016-02-27 07:02:00 +00:00
|
|
|
s = time.time()
|
|
|
|
result = f(*args, **kwargs)
|
|
|
|
e = time.time()
|
|
|
|
logmethod('running %s(%r, %r) took %.3f s',
|
|
|
|
f.__name__, args, kwargs, e - s)
|
|
|
|
return result
|
2016-02-27 07:18:24 +00:00
|
|
|
return timing
|
2016-02-27 07:02:00 +00:00
|
|
|
return anon
|
|
|
|
|
|
|
|
|
2016-02-22 20:32:16 +00:00
|
|
|
def firstfoundfile(locations):
|
|
|
|
'''return the first existing file from a list of filenames (or None)'''
|
|
|
|
for option in locations:
|
2016-02-23 21:10:10 +00:00
|
|
|
if isreadablefile(option):
|
2016-02-22 20:32:16 +00:00
|
|
|
return option
|
|
|
|
return None
|
|
|
|
|
2016-02-11 19:28:38 +00:00
|
|
|
|
2016-03-28 18:10:27 +00:00
|
|
|
def arg_isloglevel(l, defaultlevel=logging.ERROR):
|
2016-02-23 01:10:45 +00:00
|
|
|
try:
|
|
|
|
level = int(l)
|
2016-02-25 20:29:12 +00:00
|
|
|
return level
|
2016-02-23 01:10:45 +00:00
|
|
|
except ValueError:
|
2016-02-25 20:29:12 +00:00
|
|
|
pass
|
|
|
|
level = getattr(logging, l.upper(), None)
|
|
|
|
if not level:
|
2016-03-28 18:10:27 +00:00
|
|
|
level = defaultlevel
|
2016-02-23 01:10:45 +00:00
|
|
|
return level
|
|
|
|
|
|
|
|
|
2016-03-27 07:44:00 +00:00
|
|
|
def arg_isstr(s):
|
|
|
|
if isstr(s):
|
|
|
|
return s
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
2016-02-23 21:10:10 +00:00
|
|
|
def arg_isreadablefile(f):
|
|
|
|
if isreadablefile(f):
|
2016-02-23 20:18:46 +00:00
|
|
|
return f
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
2016-02-23 19:02:36 +00:00
|
|
|
def arg_isdirectory(d):
|
2016-02-23 21:10:10 +00:00
|
|
|
if os.path.isdir(d):
|
2016-02-22 21:04:03 +00:00
|
|
|
return d
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
2016-02-23 19:02:36 +00:00
|
|
|
def arg_isexecutable(f):
|
|
|
|
if isexecutable(f):
|
|
|
|
return f
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
2016-03-08 17:45:54 +00:00
|
|
|
def sameFilesystem(d0, d1):
|
|
|
|
return os.stat(d0).st_dev == os.stat(d1).st_dev
|
|
|
|
|
|
|
|
|
2016-02-19 08:54:16 +00:00
|
|
|
def stem_and_ext(name):
|
|
|
|
'''return (stem, ext) for any relative or absolute filename'''
|
|
|
|
return os.path.splitext(os.path.basename(os.path.normpath(name)))
|
|
|
|
|
|
|
|
|
2016-03-07 23:04:16 +00:00
|
|
|
def swapdirs(a, b):
|
|
|
|
'''use os.rename() to make "a" become "b"'''
|
|
|
|
if not os.path.isdir(a):
|
|
|
|
raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), a)
|
|
|
|
tname = None
|
2016-03-08 03:57:17 +00:00
|
|
|
if os.path.exists(b):
|
2016-03-07 23:04:16 +00:00
|
|
|
tdir = mkdtemp(prefix='swapdirs-', dir=opd(opa(a)))
|
|
|
|
logger.debug("Created tempdir %s.", tdir)
|
|
|
|
tname = opj(tdir, opb(b))
|
|
|
|
logger.debug("About to rename %s to %s.", b, tname)
|
|
|
|
os.rename(b, tname)
|
|
|
|
logger.debug("About to rename %s to %s.", a, b)
|
|
|
|
os.rename(a, b)
|
|
|
|
if tname:
|
|
|
|
logger.debug("About to rename %s to %s.", tname, a)
|
|
|
|
os.rename(tname, a)
|
|
|
|
logger.debug("About to remove %s.", tdir)
|
|
|
|
os.rmdir(tdir)
|
|
|
|
|
|
|
|
|
2016-03-05 01:06:55 +00:00
|
|
|
def logfilecontents(logmethod, prefix, fname):
|
|
|
|
'''log all lines of a file with a prefix '''
|
2016-03-15 04:42:07 +00:00
|
|
|
with codecs.open(fname, encoding='utf-8') as f:
|
2016-03-05 01:06:55 +00:00
|
|
|
for line in f:
|
|
|
|
logmethod("%s: %s", prefix, line.rstrip())
|
|
|
|
|
|
|
|
|
2016-03-08 06:13:23 +00:00
|
|
|
def conditionallogging(result, prefix, fname):
|
|
|
|
if logger.isEnabledFor(logging.DEBUG):
|
|
|
|
logfilecontents(logger.debug, prefix, fname) # -- always
|
|
|
|
elif logger.isEnabledFor(logging.INFO):
|
|
|
|
if result != 0:
|
|
|
|
logfilecontents(logger.info, prefix, fname) # -- error
|
|
|
|
|
|
|
|
|
2016-02-15 22:03:56 +00:00
|
|
|
def execute(cmd, stdin=None, stdout=None, stderr=None,
|
|
|
|
logdir=None, env=os.environ):
|
2016-02-19 01:23:26 +00:00
|
|
|
'''(yet another) wrapper around subprocess.Popen()
|
|
|
|
|
|
|
|
The processing tools for handling DocBook SGML, DocBook XML and Linuxdoc
|
|
|
|
all use different conventions for writing outputs. Some write into the
|
|
|
|
working directory. Others write to STDOUT. Others accept the output file
|
|
|
|
as a required option.
|
|
|
|
|
|
|
|
To allow for automation and flexibility, this wrapper function does what
|
|
|
|
most other synchronous subprocess.Popen() wrappers does, but it adds a
|
|
|
|
feature to record the STDOUT and STDERR of the executable. This is
|
|
|
|
helpful when trying to diagnose build failures of individual documents.
|
|
|
|
|
|
|
|
Required:
|
|
|
|
|
|
|
|
- cmd: (list form only; the paranoid prefer shell=False)
|
|
|
|
this must include the whole command-line
|
|
|
|
- logdir: an existing directory in which temporary log files
|
|
|
|
will be created
|
|
|
|
|
|
|
|
Optional:
|
|
|
|
|
|
|
|
- stdin: if not supplied, STDIN (FD 0) will be left as is
|
|
|
|
- stdout: if not supplied, STDOUT (FD 1) will be connected
|
|
|
|
to a named file in the logdir (and left for later inspection)
|
|
|
|
- stderr: if not supplied, STDERR (FD 2) will be connected
|
|
|
|
to a named file in the logdir (and left for later inspection)
|
|
|
|
- env: if not supplied, just use current environment
|
|
|
|
|
|
|
|
Returns: the numeric exit code of the process
|
|
|
|
|
|
|
|
Side effects:
|
|
|
|
|
|
|
|
* will probably create temporary files in logdir
|
|
|
|
* function calls wait(); process execution will intentionally block
|
|
|
|
until the child process terminates
|
|
|
|
|
|
|
|
Possible exceptions:
|
|
|
|
|
|
|
|
* if the first element of list cmd does not contain an executable,
|
|
|
|
this function will raise an AssertionError
|
|
|
|
* if logdir is not a directory, this function will raise ValueError or
|
|
|
|
IOError
|
|
|
|
* and, of course, any exceptions passed up from calling subprocess.Popen
|
|
|
|
|
|
|
|
'''
|
2016-02-15 22:03:56 +00:00
|
|
|
prefix = os.path.basename(cmd[0]) + '.' + str(os.getpid()) + '-'
|
|
|
|
|
2016-02-18 17:13:46 +00:00
|
|
|
assert isexecutable(cmd[0])
|
|
|
|
|
2016-02-17 07:42:22 +00:00
|
|
|
if logdir is None:
|
2016-02-19 01:23:26 +00:00
|
|
|
raise ValueError("logdir must be a directory, cannot be None.")
|
2016-02-17 03:52:07 +00:00
|
|
|
|
|
|
|
if not os.path.isdir(logdir):
|
|
|
|
raise IOError(errno.ENOENT, os.strerror(errno.ENOENT), logdir)
|
2016-02-15 22:03:56 +00:00
|
|
|
|
|
|
|
# -- not remapping STDIN, because that doesn't make sense here
|
2016-02-23 19:02:36 +00:00
|
|
|
mytfile = functools.partial(mkstemp, prefix=prefix, dir=logdir)
|
2016-02-15 22:03:56 +00:00
|
|
|
if stdout is None:
|
2016-02-23 19:02:36 +00:00
|
|
|
stdout, stdoutname = mytfile(suffix='.stdout')
|
|
|
|
else:
|
|
|
|
stdoutname = None
|
|
|
|
|
2016-02-15 22:03:56 +00:00
|
|
|
if stderr is None:
|
2016-02-23 19:02:36 +00:00
|
|
|
stderr, stderrname = mytfile(suffix='.stderr')
|
|
|
|
else:
|
|
|
|
stderrname = None
|
2016-02-15 22:03:56 +00:00
|
|
|
|
|
|
|
logger.debug("About to execute: %r", cmd)
|
|
|
|
proc = subprocess.Popen(cmd, shell=False, close_fds=True,
|
|
|
|
stdin=stdin, stdout=stdout, stderr=stderr,
|
|
|
|
env=env, preexec_fn=os.setsid)
|
|
|
|
result = proc.wait()
|
|
|
|
if result != 0:
|
2016-02-26 19:24:42 +00:00
|
|
|
logger.error("Non-zero exit (%s) for process: %r", result, cmd)
|
|
|
|
logger.error("Find STDOUT/STDERR in %s/%s*", logdir, prefix)
|
2016-02-23 19:02:36 +00:00
|
|
|
if isinstance(stdout, int) and stdoutname:
|
|
|
|
os.close(stdout)
|
2016-03-05 01:06:55 +00:00
|
|
|
conditionallogging(result, 'STDOUT', stdoutname)
|
2016-02-23 19:02:36 +00:00
|
|
|
if isinstance(stderr, int) and stderrname:
|
|
|
|
os.close(stderr)
|
2016-03-05 01:06:55 +00:00
|
|
|
conditionallogging(result, 'STDERR', stderrname)
|
2016-02-15 22:03:56 +00:00
|
|
|
return result
|
|
|
|
|
|
|
|
|
2016-02-23 21:10:10 +00:00
|
|
|
def isexecutable(f):
|
2016-02-19 01:23:26 +00:00
|
|
|
'''True if argument is executable'''
|
2016-02-23 21:10:10 +00:00
|
|
|
return os.path.isfile(f) and os.access(f, os.X_OK)
|
|
|
|
|
|
|
|
|
|
|
|
def isreadablefile(f):
|
|
|
|
'''True if argument is readable file'''
|
|
|
|
return os.path.isfile(f) and os.access(f, os.R_OK)
|
2016-02-12 18:31:50 +00:00
|
|
|
|
|
|
|
|
2016-03-27 07:44:00 +00:00
|
|
|
def isstr(s):
|
|
|
|
'''True if argument is stringy (unicode or string)'''
|
2016-03-27 08:02:44 +00:00
|
|
|
try:
|
|
|
|
unicode
|
|
|
|
stringy = (str, unicode)
|
|
|
|
except NameError:
|
2016-04-02 18:58:19 +00:00
|
|
|
stringy = (str,) # -- python3
|
2016-03-27 08:02:44 +00:00
|
|
|
return isinstance(s, stringy)
|
2016-03-27 07:44:00 +00:00
|
|
|
|
|
|
|
|
2016-02-12 18:31:50 +00:00
|
|
|
def which(program):
|
|
|
|
'''return None or the full path to an executable (respecting $PATH)
|
|
|
|
http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python/377028#377028
|
|
|
|
'''
|
|
|
|
fpath, fname = os.path.split(program)
|
2016-02-18 17:13:46 +00:00
|
|
|
if fpath and isexecutable(program):
|
2016-02-12 18:31:50 +00:00
|
|
|
return program
|
|
|
|
else:
|
|
|
|
for path in os.environ["PATH"].split(os.pathsep):
|
|
|
|
path = path.strip('"')
|
|
|
|
sut = os.path.join(path, program)
|
2016-02-18 17:13:46 +00:00
|
|
|
if isexecutable(sut):
|
2016-02-12 18:31:50 +00:00
|
|
|
return sut
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
2016-04-02 17:45:53 +00:00
|
|
|
def writemd5sums(fname, md5s, header=None):
|
|
|
|
'''write an MD5SUM file from [(filename, MD5), ...]'''
|
|
|
|
with codecs.open(fname, 'w', encoding='utf-8') as file:
|
|
|
|
if header:
|
|
|
|
print(header, file=file)
|
2016-04-02 17:46:43 +00:00
|
|
|
for fname, hashval in sorted(md5s.items()):
|
2016-04-02 17:45:53 +00:00
|
|
|
print(hashval + ' ' + fname, file=file)
|
|
|
|
|
|
|
|
|
|
|
|
def md5file(name):
|
|
|
|
'''return MD5 hash for a single file name'''
|
|
|
|
with open(name, 'rb') as f:
|
|
|
|
bs = f.read()
|
|
|
|
md5 = hashlib.md5(bs).hexdigest()
|
|
|
|
try:
|
|
|
|
md5 = unicode(md5)
|
|
|
|
except NameError:
|
|
|
|
pass # -- python3
|
|
|
|
return md5
|
|
|
|
|
|
|
|
|
2016-02-17 19:31:08 +00:00
|
|
|
def statfile(name):
|
2016-02-19 01:23:26 +00:00
|
|
|
'''return posix.stat_result (or None) for a single file name'''
|
2016-02-17 19:31:08 +00:00
|
|
|
try:
|
2016-02-29 19:32:01 +00:00
|
|
|
st = os.lstat(name)
|
2016-02-17 19:31:08 +00:00
|
|
|
except OSError as e:
|
|
|
|
if e.errno != errno.ENOENT:
|
|
|
|
raise e
|
2016-02-29 19:32:01 +00:00
|
|
|
st = None
|
2016-02-17 19:31:08 +00:00
|
|
|
return st
|
|
|
|
|
|
|
|
|
2016-04-02 17:45:53 +00:00
|
|
|
def md5files(name, relative=None):
|
|
|
|
'''get all of the MD5s for files from here downtree'''
|
|
|
|
return fileinfo(name, relative=relative, func=md5file)
|
|
|
|
|
2016-02-17 19:18:47 +00:00
|
|
|
def statfiles(name, relative=None):
|
2016-04-02 17:45:53 +00:00
|
|
|
'''
|
|
|
|
>>> statfiles('./docs/x509').keys()
|
|
|
|
['./docs/x509/tutorial.rst', './docs/x509/reference.rst', './docs/x509/index.rst']
|
|
|
|
>>> statfiles('./docs/x509', relative='./').keys()
|
|
|
|
['docs/x509/reference.rst', 'docs/x509/tutorial.rst', 'docs/x509/index.rst']
|
|
|
|
>>> statfiles('./docs/x509', relative='./docs/x509/').keys()
|
|
|
|
['index.rst', 'tutorial.rst', 'reference.rst']
|
|
|
|
'''
|
|
|
|
return fileinfo(name, relative=relative, func=statfile)
|
|
|
|
|
|
|
|
def fileinfo(name, relative=None, func=statfile):
|
2016-02-19 01:23:26 +00:00
|
|
|
'''return a dict() with keys being filenames and posix.stat_result values
|
|
|
|
|
|
|
|
Required:
|
|
|
|
|
2016-02-19 07:07:14 +00:00
|
|
|
name: the name should be an existing file, but accessing filesystems
|
2016-02-19 01:23:26 +00:00
|
|
|
can be a racy proposition, so if the name is ENOENT, returns an
|
|
|
|
empty dict()
|
|
|
|
if name is a directory, os.walk() over the entire subtree and
|
|
|
|
record and return all stat() results
|
|
|
|
|
|
|
|
Optional:
|
|
|
|
|
|
|
|
relative: if the filenames in the keys should be relative some other
|
|
|
|
directory, then supply that path here (see examples)
|
|
|
|
|
|
|
|
|
|
|
|
Bugs:
|
|
|
|
Dealing with filesystems is always potentially a racy affair. They go
|
|
|
|
out for lunch sometimes. They don't call. They don't write. But, at
|
|
|
|
least we can try to rely on them as best we can--mostly, by just
|
|
|
|
excluding any files (in the output dict()) which did not return a valid
|
|
|
|
posix.stat_result.
|
|
|
|
'''
|
2016-04-02 17:45:53 +00:00
|
|
|
info = dict()
|
2016-02-18 02:16:20 +00:00
|
|
|
if not os.path.exists(name):
|
2016-04-02 17:45:53 +00:00
|
|
|
return info
|
2016-02-17 19:18:47 +00:00
|
|
|
if not os.path.isdir(name):
|
|
|
|
if relative:
|
|
|
|
relpath = os.path.relpath(name, start=relative)
|
|
|
|
else:
|
|
|
|
relpath = name
|
2016-04-02 17:45:53 +00:00
|
|
|
info[relpath] = func(name)
|
|
|
|
if info[relpath] is None:
|
|
|
|
del info[relpath]
|
2016-02-17 19:18:47 +00:00
|
|
|
else:
|
|
|
|
for root, dirs, files in os.walk(name):
|
2016-02-19 05:55:56 +00:00
|
|
|
inodes = list()
|
|
|
|
inodes.extend(dirs)
|
|
|
|
inodes.extend(files)
|
|
|
|
for x in inodes:
|
2016-02-17 19:18:47 +00:00
|
|
|
foundpath = os.path.join(root, x)
|
2016-04-02 05:17:53 +00:00
|
|
|
if os.path.isdir(foundpath):
|
|
|
|
continue
|
2016-02-17 19:18:47 +00:00
|
|
|
if relative:
|
|
|
|
relpath = os.path.relpath(foundpath, start=relative)
|
|
|
|
else:
|
|
|
|
relpath = foundpath
|
2016-04-02 17:45:53 +00:00
|
|
|
info[relpath] = func(foundpath)
|
|
|
|
if info[relpath] is None:
|
|
|
|
del info[relpath]
|
|
|
|
return info
|
2016-02-17 07:42:22 +00:00
|
|
|
|
2016-02-11 03:22:23 +00:00
|
|
|
#
|
|
|
|
# -- end of file
|