Merge pull request #12 from martin-a-brown/mabrown/version-0.7.15

bump version to 0.7.15

.travis.yml

@@ -7,7 +7,7 @@ before_install:
 python:
   - "2.7"
   - "3.4"
-script: nosetests --cover-erase --with-coverage --cover-package tldp -- tests/long_driver.py tests/
+script: nosetests --cover-erase --with-coverage --cover-package tldp -- tests/long_driver.py tests/long_inventory.py tests/
 
 # -- comments on install set on an Ubuntu system:
 #    Here is the full set of packages that need to be installed in order for

ChangeLog

@@ -0,0 +1,220 @@
+
+2016-05-13 Martin A. Brown <martin@linux-ip.net>
+  * bumping version to tldp-0.7.13
+  * accommodate root-run tests (used by Deb-O-Matic)
+
+2016-04-30 Martin A. Brown <martin@linux-ip.net>
+  * bumping version to tldp-0.7.12
+  * adding ChangeLog (this file)
+  * cosmetic changes; deduplication of test data, copyright in many files
+  * add contrib/debian-release.sh
+  * put version number in tldp/__init__.py
+  * generate specfile after tagging, using contrib/rpm-release.py
+  * Debian packaging issues larger addressed
+
+2016-04-21 Martin A. Brown <martin@linux-ip.net>
+  * bumping version to tldp-0.7.7
+  * Debian packaging attempt #1, created build with 'native' source format
+    which will not be accepted
+  * add debian/copyright file
+  * ldptool manpage (sphinx-generated for Debian; statically installed in RPM)
+  * switch --detail reporting to use predictable DOCTYPE and STATUSTYPE names
+
+2016-04-09 Martin A. Brown <martin@linux-ip.net>
+  * bumping version to 0.7.5
+  * remove 'random' text from .LDP-source-MD5SUMS
+  * remove the --builddir if empty after complete run
+
+2016-04-02 Martin A. Brown <martin@linux-ip.net>
+  * bumping version to 0.7.2
+  * using filesystem age for determining build need will not work; switch
+    to using content hash (MD5) to determine whether a rebuild is necessary or
+    not
+  * create .LDP-source-MD5SUMS in each output directory that lists all of
+    the hashes of the source files used to create that output directory
+  * remove testing and references to statfiles() and supporting friends
+  * add a 'lifecycle' test to the testing suite
+  * report on running success and failure counts during the run (to allow
+    interruptability if the user wishes)
+
+2016-03-28 Martin A. Brown <martin@linux-ip.net>
+  * bumping version to 0.7.0
+  * support better handling of --verbose; --verbose yes, --verbose false
+  * update and improve documentation in stock configuration file
+  * provide better feedback on directory existence (or not) rather than
+    silently doing something unpredicable
+
+2016-03-27 Martin A. Brown <martin@linux-ip.net>
+  * bumping version to 0.6.7
+  * correct situation where publish() was not propagating errors returned
+    from the build() function; add test
+  * add broken example Docbook 4 XML file to test suite
+  * use unicode_literals in all testing code, too
+
+2016-03-24 Martin A. Brown <martin@linux-ip.net>
+  * bumping version to 0.6.2
+  * fix all sorts of runtime requirements to build under Ubuntu
+    and run the full test suite on Travis CI
+
+2016-03-15 Martin A. Brown <martin@linux-ip.net>
+  * bumping version to 0.6.0
+  * full support for Python3, all unicode-ified and happy
+  * add test to fall back to iso-8859-1 for SGML docs
+  * success testing with tox under Python 2.7 and 3.4
+
+2016-03-14 Martin A. Brown <martin@linux-ip.net>
+  * bumping version to 0.5.5
+  * use sgmlcheck for Linuxdoc sources
+  * adjust reporting of discovered documents
+  * use context to prevent more FD leakage
+  * begin changes to support Python3; e.g. io.StringIO, absolute_import
+    unicode changes, lots of codecs.open(), unicode_literals, 
+
+2016-03-11 Martin A. Brown <martin@linux-ip.net>
+  * handle EPIPE and INT with signal.SIG_DFL
+
+2016-03-10 Martin A. Brown <martin@linux-ip.net>
+  * bumping version to 0.5.3
+  * create long running tests that exercise more of the code in the likely
+    way that a user would use the utility
+  * add testing for Docbook 5 XML
+  * improve look and consistency for --list (--detail) output
+  * improve README.rst
+
+2016-03-09 Martin A. Brown <martin@linux-ip.net>
+  * remove unused markdown and rst skeleton processors
+  * pass **kwargs through all processor tools
+
+2016-03-07 Martin A. Brown <martin@linux-ip.net>
+  * add support for --builddir, ensure that --builddir is on the same
+    filesystem as --pubdir
+  * add new option --publish; can't replace a directory atomically, but
+    get as close as possible by swapping the newly built output (from
+    --builddir) with the old one (formerly in --pubdir)
+  * switch to using 'return os.EX_OK' from functions in driver.py that
+    can be tested and/or wrapped in sys.exit(function(args))
+  * testing improvements for Asciidoc and driver.py
+
+2016-03-06 Martin A. Brown <martin@linux-ip.net>
+  * provide user-discoverable support for --doctypes and --statustypes
+  * correct removal of Docbook4XML generated source document during build
+
+2016-03-05 Martin A. Brown <martin@linux-ip.net>
+  * use a simplified technique (arbitrary attributes on function objects)
+    to generate the DAG used for topological sorting and build order
+    generation (thanks to Python mailing lists for the idea)
+
+2016-03-04 Martin A. Brown <martin@linux-ip.net>
+  * bumping version to 0.4.8
+  * add FO generation XSL
+  * do not set a system default for --sourcedir / --pubdir (user must
+    specify, somehow)
+  * DocBook5/DocBook4: process xincludes before validation with xmllint
+  * add support for AsciiDoc detection and processing
+
+2016-03-03 Martin A. Brown <martin@linux-ip.net>
+  * validate all documents (where possible) before processing
+  * provide support for DocBook 5.0 (XML)
+  * correct --loglevel handling in driver.py (finally works properly!)
+  * complete support for --script output
+
+2016-03-02 Martin A. Brown <martin@linux-ip.net>
+  * bumping version to 0.4.5
+  * fix handling of STEMs which contain a '.' in the name
+  * review signature identification in each DOCTYPE processor and 
+    validate and reconcile errors with PUBLIC / SYSTEM identifiers
+    for the SGML and XML declarations
+  * make sure that build() exits non-zero if ANY build fails
+
+2016-03-01 Martin A. Brown <martin@linux-ip.net>
+  * bumping version to 0.4.2
+  * support a system configuration file /etc/ldptool
+  * add entry points and make first full installable build
+  * allow empty OutputDirectory() object
+  * begin overhauling the porcelain in driver.py
+
+2016-02-29 Martin A. Brown <martin@linux-ip.net>
+  * overhaul generation of inventory object from sources/outputs
+  * add command-line features and options; actions in particular
+  * continue improving coverage, at 100% on utils.py
+  * complete CascadingConfig object creation
+
+2016-02-26 Martin A. Brown <martin@linux-ip.net>
+  * generate a DAG for each processor class, so dependencies can
+    be localized (controlled, abstracted) to each processor class
+  * use topological sort of the DAG to drive generation of the shellscript,
+    which leads to massive simplification of the generate() method
+  * user can specify explicit file to process
+  * better PDF generation logic (relying on jw)
+  * provide support for --script outputs (logical equiv. of --dryrun)
+  * if a document processor is missing prerequisites, gripe to logging
+    and skip to the next document
+  * support a SourceDocument named by its directory
+  * add timing to each processor (some documents take minutes to process,
+    others just a few seconds; good for users trying to understand which...)
+
+2016-02-25 Martin A. Brown <martin@linux-ip.net>
+  * overhaul where and how logging module gets called; driver.py is main
+  * adding --skip feature; can skip STEM, DOCTYPE or STATUSTYPE
+  * automatically detect configuration fragments in document processors
+    with object inspection
+
+2016-02-23 Martin A. Brown <martin@linux-ip.net>
+  * add support for --detail (and --verbose) for both source and output docs
+  * pass args into all driver functions
+  * get rid of platform.py and references (not necessary any longer)
+  * fix FD leakage in function execute() and add test case (prevent reversion)
+    (and start switching to contextlib 'with' usage to avoid in future)
+  * start generalizing the build process for all doctypes in common.py
+  * move all generic functionality into BaseDoctype object
+  * revise fundamental execution approach; generate a shellscript (which can
+    be executed or simply printed)
+  * make logging readability improvements: clarity, relevance and alignment
+
+2016-02-22 Martin A. Brown <martin@linux-ip.net>
+  * adding ArgumentParser wrapper so can support config file + envars
+  * all sorts of work for support cascading configuration
+  * allow each processor to have its own configuration fragment, e.g.
+    --docbook4xml-xmllint; owned by the Docbook4XML object
+  * add support for --dump-cfg, --dump-env, --dump-cli, --debug-options
+  * adding the license text (MIT) and all of that stuff
+  * creating and fixing the setup.py
+
+2016-02-19 Martin A. Brown <martin@linux-ip.net>
+
+2016-02-18 Martin A. Brown <martin@linux-ip.net>
+  * process and report on documents in case-insensitive stem-sorted order
+  * add many docstrings for internal usage
+  * move all source directory scanning logic out of the SourceCollection
+    object; easier to test and simpler to understand
+
+2016-02-17 Martin A. Brown <martin@linux-ip.net>
+  * add logic for testing file age, assuming a fresh checkout of the
+    source documents; use filesystem age to determine whether or not
+    a document rebuild is necessary
+  * initial support for driver.py (eventually, the main user entry point
+    and inventory.py (for managing the identification and pairing of
+    source and output documents)
+
+2016-02-16 Martin A. Brown <martin@linux-ip.net>
+  * adding tons of testing for document types, edge cases, duplicate
+    stems, sample valid and broken documents
+
+2016-02-15 Martin A. Brown <martin@linux-ip.net>
+  * first processor, Linuxdoc, reaches success
+  * provide better separation between a SourceCollection and the
+    individual SourceDocuments; analogously, between OutputDirectory
+    and OutputCollection
+  * provide similar dict-like behaviour for SourceCollection and
+    OutputCollection (which is known to the user as --pubdir)
+
+2016-02-12 Martin A. Brown <martin@linux-ip.net>
+  * first processor, Linuxdoc, fleshed out, created (failed)
+  * generate skeletons for other supported source document formats
+  * automate detection of source document format; add initial testing tools
+
+2016-02-11 Martin A. Brown <martin@linux-ip.net>
+  * core source collection and output directory scanning complete
+
+2016-02-10 Martin A. Brown <martin@linux-ip.net>
+  * initial commit and basic beginnings

LICENSE

@@ -1,3 +1,5 @@
+Copyright (c) 2016, Linux Documentation Project
+
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation files
 (the "Software"), to deal in the Software without restriction,

MANIFEST.in

@@ -1,3 +1,4 @@
 include README.rst
 recursive-include etc *
+recursive-include docs *
 recursive-include contrib *

NOTES.rst

@@ -0,0 +1,18 @@
+Notes to future self
+++++++++++++++++++++
+
+To release a new version for different software consumers.
+
+  * commit all of the changes you want
+  * bump version in tldp/__init__.py
+  * adjust debian/changelog in accordance with Debian policy
+    N.B. the version must match what you put in tldp/__init__.py
+  * run 'python contrib/rpm-release.py' which will regenerate a
+    contrib/tldp.spec with the correct version
+  * commit debian/changelog tldp/__init__.py and contrib/tldp.spec
+  * tag the release
+  * run 'git push origin master --tags'
+  * run 'python setup.py sdist upload -r pypi'
+  * run 'bash contrib/debian-release.py' (on a Debian-ish box)
+
+

README.rst

@@ -4,6 +4,10 @@ tldp - tools for publishing from TLDP sources
 .. image:: https://api.travis-ci.org/martin-a-brown/python-tldp.svg
     :target: https://github.com/tLDP/python-tldp
 
+.. image:: http://img.shields.io/badge/license-MIT-brightgreen.svg 
+    :target: http://opensource.org/licenses/MIT
+    :alt: MIT license
+
 This package was written for the Linux Documentation Project (TLDP) to help
 with management and publication automation of source documents.  The primary
 interface provided is a command-line tool caalled `ldptool`.  The canonical
@@ -334,9 +338,11 @@ called ldp-docbook-{xsl,dsssl}.  There aren't any such packages for RPM (yet).
 
 Supported Python versions
 -------------------------
-This package was built and used against Python-2.7.8 (OpenSUS) and
-Python-2.7.6 (Ubuntu).  It has been tested (success for test suite) against
-Python-3.4.1 and lightly used against Python-3.4.1.
+This package was developed against Python-2.7.8 and Python-3.4.1 (on
+OpenSUSE).  It has been used on Python-2.7.6 (Ubuntu-14.04) and Python-3.4.2 and Python-2.7.9 (on Debian 8).
+
+Continuous Integration testing information and coverage can be reviewed at
+`this project's Travis CI page <https://travis-ci.org/martin-a-brown/python-tldp/>`_.
 
 
 Installation
@@ -347,25 +353,39 @@ requires a large number of other packages, most of which are outside of the
 Python ecosystem.  There's room for improvement here, but here are a few
 tidbits.
 
-Build an RPM:::
+Build an RPM::
 
   python setup.py sdist && rpmbuild -ta ./dist/python-tldp-${VERSION}.tar.gz
 
-There's a file, `contrib/tldp.spec`, which makes a few changes to the
-setuptools stock-generated specfile.  Specifically, the package gets named
-`python-tldp` instead of `tldp` and the configuration file is marked
-`%config(noreplace)`.
+There's a generated file, `contrib/tldp.spec`, which makes a few changes to the
+setuptools stock-generated specfile.  It adds the dependencies, marks the
+configuration file as %config(noreplace), adds a manpage and names the binary
+package `python-tldp`.
 
-I know less about packaging for Debian.  Relying on python-stdeb yields a
-working and usable Debian package which has been tested out on an Ubuntu
-14.04.3 system.
+Build a DEB::
 
-Build a DEB:::
+Check to see if the package is available from upstream.  It may be included in
+the Debian repositories already::
+
+  apt-cache search tldp
+
+The quick and dirty way is as follows::
 
   python setup.py --command-packages=stdeb.command bdist_deb
 
-I have not tried installing the package in a virtualenv or with pip.  If you
-try that, please let me know any problems you encounter.
+But, there is also a `debian` directory.  If you are working straight from the
+git checkout, you should be able to generate an installable (unsigned) Debian
+package with::
+
+  bash contrib/debian-release.sh -us -uc
+
+Install using pip:
+
+Unknown.  Because the tool relies so heavily on system-installed non-Python
+tools, I have not bothered to try installing the package using pip.  It should
+work equivalently as well as running the program straight from a checkout.
+If you learn anything here or have suggestions, for me, please feel free to
+send them along.
 
 
 Links

TODO

@@ -1,6 +1,12 @@
 python-tldp TODO
 ================
 
+Bugs
+----
+
+  * when running --sourcedir $FILE, the error message is TERRIBLE;
+    fix it;
+
 user-visible needs
 ------------------
 

contrib/debian-release.sh

@@ -0,0 +1,26 @@
+#! /bin/bash
+#
+#
+
+set -e
+set -x
+set -o pipefail
+
+PACKAGE=$(dpkg-parsechangelog | awk '/Source:/{print $2}')
+VERSION=$(dpkg-parsechangelog | awk -F'[- ]' '/Version:/{print $2}')
+
+PREFIX="${PACKAGE}-${VERSION}"
+TARBALL="../${PACKAGE}_${VERSION}.orig.tar.xz"
+
+git archive \
+  --format tar \
+  --prefix "${PREFIX}/" \
+  "${PREFIX}" \
+  | xz \
+    --compress \
+    --to-stdout \
+  > "${TARBALL}"
+
+exec debuild "$@"
+
+# -- end of file

contrib/rpm-release.py

@@ -0,0 +1,29 @@
+#! /usr/bin/python
+#
+#
+
+from __future__ import print_function
+
+import os
+import sys
+
+opd = os.path.dirname
+opj = os.path.join
+
+sys.path.insert(0, opd(opd(__file__)))
+from tldp import VERSION
+
+fin = open(opj(opd(__file__), 'tldp.spec.in'))
+fout = open(opj(opd(__file__), 'tldp.spec'), 'w')
+
+def transform(mapping, text):
+    for tag, replacement in mapping.items():
+        text = text.replace(tag, replacement)
+    return text
+
+subst = {'@VERSION@': VERSION}
+print(subst)
+
+fout.write(transform(subst, fin.read()))
+
+# -- end of file

contrib/tldp.spec

@@ -1,11 +1,11 @@
 %define sourcename tldp
 %define name python-tldp
-%define version 0.6.6
-%define unmangled_version 0.6.6
-%define unmangled_version 0.6.6
+%define version 0.7.15
+%define unmangled_version 0.7.15
+%define unmangled_version 0.7.15
 %define release 1
 
-Summary: processing tools for Asciidoc, DocBook XML, DocBook SGML and Linuxdoc
+Summary: automatic publishing tool for DocBook, Linuxdoc and Asciidoc
 Name: %{name}
 Version: %{version}
 Release: %{release}
@@ -31,8 +31,8 @@ Requires: libxslt-tools
 Requires: python-networkx
 
 %description
-tldp - processing tools for Asciidoc, DocBook XML, DocBook SGML and Linuxdoc
-============================================================================
+tldp - automatic publishing tool for DocBook, Linuxdoc and Asciidoc
+===================================================================
 A toolset for publishing multiple output formats (PDF, text, chunked HTML and
 single-page HTML) from each source document in one of the supported formats.
 
@@ -54,6 +54,7 @@ python setup.py build
 
 %install
 python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES
+install -D --mode 0644 docs/ldptool.1 %{buildroot}%{_mandir}/man1/ldptool.1
 perl -pi -e 's,(/etc/ldptool/ldptool.ini),%config(noreplace) $1,' INSTALLED_FILES
 
 %clean
@@ -61,3 +62,4 @@ rm -rf $RPM_BUILD_ROOT
 
 %files -f INSTALLED_FILES
 %defattr(-,root,root)
+%{_mandir}/man1/ldptool.1*

contrib/tldp.spec.in

@@ -0,0 +1,65 @@
+%define sourcename tldp
+%define name python-tldp
+%define version @VERSION@
+%define unmangled_version @VERSION@
+%define unmangled_version @VERSION@
+%define release 1
+
+Summary: automatic publishing tool for DocBook, Linuxdoc and Asciidoc
+Name: %{name}
+Version: %{version}
+Release: %{release}
+Source0: %{sourcename}-%{unmangled_version}.tar.gz
+License: MIT
+Group: Development/Libraries
+BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
+Prefix: %{_prefix}
+BuildArch: noarch
+Vendor: Martin A. Brown <martin@linux-ip.net>
+BuildRequires: python-setuptools
+Requires: asciidoc
+Requires: jing
+Requires: htmldoc
+Requires: sgmltool
+Requires: openjade
+Requires: docbook-utils
+Requires: docbook-utils-minimal
+Requires: docbook-dsssl-stylesheets
+Requires: docbook-xsl-stylesheets
+Requires: docbook5-xsl-stylesheets
+Requires: libxslt-tools
+Requires: python-networkx
+
+%description
+tldp - automatic publishing tool for DocBook, Linuxdoc and Asciidoc
+===================================================================
+A toolset for publishing multiple output formats (PDF, text, chunked HTML and
+single-page HTML) from each source document in one of the supported formats.
+
+ * Asciidoc
+ * Linuxdoc
+ * Docbook SGML 3.x (though deprecated, please no new submissions)
+ * Docbook SGML 4.x
+ * Docbook XML 4.x
+ * Docbook XML 5.x (basic support, as of 2016-03-10)
+
+TLDP = The Linux Documentation Project.
+
+
+%prep
+%setup -n %{sourcename}-%{unmangled_version}
+
+%build
+python setup.py build
+
+%install
+python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES
+install -D --mode 0644 docs/ldptool.1 %{buildroot}%{_mandir}/man1/ldptool.1
+perl -pi -e 's,(/etc/ldptool/ldptool.ini),%config(noreplace) $1,' INSTALLED_FILES
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+%files -f INSTALLED_FILES
+%defattr(-,root,root)
+%{_mandir}/man1/ldptool.1*

debian/changelog

@@ -0,0 +1,21 @@
+tldp (0.7.15-1) unstable; urgency=low
+
+  * support Python3.8+: fix import for MutableMapping and other minor fixes
+
+tldp (0.7.14-1) unstable; urgency=low
+
+  * Add --version option.
+
+ -- Martin A. Brown <martin@linux-ip.net>  Mon, 16 May 2016 16:54:47 +0000
+
+tldp (0.7.13-1) unstable; urgency=low
+
+  * Fix testsuite when run as root (Closes: #824201).
+
+ -- Martin A. Brown <martin@linux-ip.net>  Fri, 13 May 2016 16:28:22 +0000
+
+tldp (0.7.12-1) unstable; urgency=low
+
+  * Initial release (Closes: #822181)
+
+ -- Martin A. Brown <martin@linux-ip.net>  Wed, 27 Apr 2016 17:09:56 +0000

debian/clean

@@ -0,0 +1,4 @@
+tldp.egg-info/
+docs/_build/
+.coverage/
+.tox/

debian/compat

@@ -0,0 +1 @@
+9

debian/control

@@ -0,0 +1,55 @@
+Source: tldp
+Maintainer: Martin A. Brown <martin@linux-ip.net>
+Section: text
+X-Python3-Version: >= 3.4
+Priority: optional
+Homepage: https://github.com/tLDP/python-tldp
+Build-Depends: debhelper (>= 9),
+               dh-python,
+               python3-all,
+               python3-networkx,
+               python3-nose,
+               python3-coverage,
+               python3-setuptools,
+               python3-sphinx,
+               htmldoc,
+               fop,
+               jing,
+               sgml2x,
+               xsltproc,
+               asciidoc,
+               docbook,
+               docbook5-xml,
+               docbook-xsl-ns,
+               linuxdoc-tools-latex,
+               linuxdoc-tools-text,
+               ldp-docbook-xsl,
+               ldp-docbook-dsssl,
+               html2text
+Standards-Version: 3.9.8
+Vcs-Git: https://github.com/tLDP/python-tldp.git
+Vcs-Browser: https://github.com/tLDP/python-tldp
+
+Package: python3-tldp
+Architecture: all
+Depends: ${misc:Depends},
+         ${python3:Depends},
+         fop,
+         jing,
+         xsltproc,
+         docbook,
+         docbook5-xml,
+         docbook-xsl-ns,
+         htmldoc,
+         html2text,
+         sgml2x,
+         asciidoc,
+         linuxdoc-tools-latex,
+         linuxdoc-tools-text,
+         ldp-docbook-xsl,
+         ldp-docbook-dsssl
+Description: automatic publishing tool for DocBook, Linuxdoc and Asciidoc
+ The Linux Documentation Project (TLDP) stores hundreds of documents in
+ DocBook SGML, DocBook XML, Linuxdoc and Asciidoc formats.  This tool
+ automatically detects the source format and generates a directory containing
+ chunked and single-page HTML, a PDF and a plain text output.

debian/copyright

@@ -0,0 +1,71 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: python-tldp
+Upstream-Contact:  Martin A. Brown <martin@linux-ip.net>
+Source: https://github.com/tLDP/python-tldp
+
+Files: *
+Copyright: 2016 Linux Documentation Project
+License: MIT
+
+Files: extras/dsssl/ldp.dsl
+Copyright: 2000-2003  - Greg Ferguson (gferg@metalab.unc.edu)
+License: GPL-2.0+
+
+Files: extras/xsl/tldp-*.xsl extras/css/style.css
+Copyright: 2000-2002  - David Horton (dhorton@speakeasy.net)
+License: GFDL-1.2
+
+Files: tests/sample-documents/DocBook-4.2-WHYNOT/images/* tests/sample-documents/DocBookSGML-Larger/images/bullet.png
+Copyright: Copyright (C) 2011-2012 O'Reilly Media
+License: MIT
+
+Files: extras/collateindex.pl
+Copyright: 1997-2001 Norman Walsh
+License: MIT
+ 
+License: MIT
+ Permission is hereby granted, free of charge, to any person obtaining a
+ copy of this software and associated documentation files (the "Software"),
+ to deal in the Software without restriction, including without limitation
+ the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ and/or sell copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following conditions:
+ .
+ The above copyright notice and this permission notice shall be included
+ in all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+License: GPL-2.0+
+ This package is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+ .
+ This package is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>
+ .
+ On Debian systems, the complete text of the GNU General
+ Public License version 2 can be found in "/usr/share/common-licenses/GPL-2".
+
+License: GFDL-1.2
+ Permission is granted to copy, distribute and/or modify this document
+ under the terms of the GNU Free Documentation License, Version 1.2
+ or any later version published by the Free Software Foundation;
+ with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts.
+ A copy of the license is included in the section entitled "GNU
+ Free Documentation License".
+ .
+ On Debian systems, the complete text of the GFDL-1.2 can be found in
+ /usr/share/common-licenses/GFDL-1.2

debian/rules

@@ -0,0 +1,11 @@
+#!/usr/bin/make -f
+# export PYBUILD_VERBOSE=1
+export PYBUILD_NAME=tldp
+
+%:
+	dh $@ --with=python3 --buildsystem=pybuild
+
+override_dh_installman:
+	(cd docs && \
+	    sphinx-build -b man -d _build/doctrees . _build/man)
+	dh_installman docs/_build/man/ldptool.1

debian/source/format

@@ -0,0 +1 @@
+3.0 (quilt)

debian/upstream/metadata

@@ -0,0 +1,4 @@
+Bug-Database: https://github.com/tLDP/LDP/issues
+Contact: discuss@en.tldp.org
+Name: python-tldp
+Repository: https://github.com/tLDP/LDP

debian/watch

@@ -0,0 +1,3 @@
+version=3
+opts=uversionmangle=s/(rc|a|b|c)/~$1/ \
+https://pypi.debian.net/tldp/tldp-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))

docs/conf.py

@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+#
+# tox documentation build configuration file, created by
+# sphinx-quickstart on Fri Nov  9 19:00:14 2012.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['sphinx.ext.autodoc']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'ldptool-man'
+
+# General information about the project.
+project = u'ldptool'
+copyright = u'Manual page (C) 2016, Linux Documentation Project'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '1.9.2'
+# The full version, including alpha/beta/rc tags.
+release = '1.9.2'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['_build']
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'default'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'ldptooldoc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = []
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('ldptool-man', 'ldptool', u'DocBook, Linuxdoc and Asciidoc build/publishing tool.',
+     [u'Martin A. Brown <martin@linux-ip.net>',], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output ------------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+  ('ldptool-man', 'ldptool', u'ldptool(1)',
+   u'Martin A. Brown', 'ldptool', 'DocBook, Linuxdoc and Asciidoc build/publishing tool.',
+   'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'

docs/ldptool-man.rst

@@ -0,0 +1,380 @@
+:orphan:
+
+ldptool manual page
+===================
+
+Synopsis
+--------
+
+**ldptool** [*options*]  [*pathname* [...]]
+
+
+Description
+-----------
+
+:program:`ldptool` creates chunked HTML, single-page HTML, PDF and plain text
+outputs for each source document it is passed as a *pathname*.  See
+`Source document discovery`_.  It will compare the source document and output
+document and rebuild an output only if the content has changed.
+
+If it is not passed any arguments, `ldptool` will collect all of the
+directories specified with the --sourcedir option and scan through these
+directories looking for valid source documents.
+
+The action taken depends on the options passed to the utility.  If no options
+are passed, then the default `--build` action will be attempted.  The options
+controlling the overall program are described in the sections `Action
+options`_ and `Main options`_.  All other options are relegated to the tail of
+the manpage, because they are merely configurables for individual document
+processors.
+
+The `ldptool` can:
+
+- generate an inventory from multiple source directories (`--sourcedir`)
+- crawl through a single output collection (`--pubdir`)
+- match the sources to the outputs (based on document stem name)
+- describe the collection by type and status (`--summary`)
+- list out individual document type and status (`--list`)
+- describe supported source formats (`--formats`)
+- describe the meaning of document status (`--statustypes`)
+- build the expected (non-configurable) set of outputs (`--build`)
+- build and publish the outputs (`--publish`)
+- produce runnable shell script to STDOUT (`--script`)
+- generate configuration files that it can then take as input
+
+
+Action options
+--------------
+-h, --help
+   show a help message and exit
+
+-V, --version
+   print out the version number
+
+-b, --build
+   Build LDP documentation into the `--builddir` and exit.
+   This is the default action if no other action is specified.
+
+-p, --publish
+   Build LDP documentation into the `--builddir`.  If all builds are
+   successful, then copy the result for each source document into the
+   `--pubdir`, effectively replacing (and deleting) the older documents;
+   finally, remove `--builddir`, if empty.
+
+-S, --script
+   Print a runnable bash script to STDOUT.  This will produce a
+   shell script showing what would be executed upon `--build`.
+
+-l, --detail, --list
+   Examine the various SOURCEDIRs and the PUBDIR and generate a report
+   showing the FORMAT of the source document and STATUS of the document.
+   Add the `--verbose` flag for more information.
+
+-t, --summary
+   Examine the various SOURCEDIRs and the PUBDIR and generate a short
+   report summarizing documents by STATUS and by DOCTYPE.  Add the
+   `--verbose` flag for more information.
+
+-T, --doctypes, --formats, --format, --list-doctypes, --list-formats
+   List the supported DOCTYPEs; there is one processor for each DOCTYPE.
+
+--statustypes, --list-statustypes
+   List the possible document STATUS types.  There are only seven basic status
+   types, but several synonyms and groups of STATUS types (internally called
+   'classes').
+
+Main options
+------------
+-s, --sourcedir, --source-dir, --source-directory SOURCEDIR (default: None)
+   Specify the name of a SOURCEDIR which contains source documents.  See
+   also `Source document discovery`_.
+
+   The `--sourcedir` option may be used more than once.
+
+-o, --pubdir, --output, --outputdir, --outdir PUBDIR (default: None)
+   Specify the the name of a PUBDIR.  Used as the publication if the build
+   succeeds.  When `--publish` is used with `--pubdir`, the output of
+   a successful document build will be used to replace any existing document
+   output directory in PUBDIR.
+
+-d, --builddir, --build-dir, --build-directory BUILDDIR (default: 'ldptool-build')
+   Specify the name of a BUILDDIR.  A scratch directory used to build each
+   source document; directory is temporary and will be removed if the
+   build succeeds AND `--publish` has been requested.  Under the `--build` 
+   action, all output directories and contents remain in the BUILDDIR for
+   inspection.
+
+--verbose [True | False] (default: False)
+   Provide more information in --list and --detail actions.  The option can
+   be thrown without an argument which is equivalent to True.  To allow the
+   CLI to supersede environment or configuration file values, `--verbose
+   false` is also supported.
+
+--skip [STEM | DOCTYPE | STATUS]
+   Specify a source document name, document type or document status to skip
+   during processing.  Each document is known by its STEM (see also `Source
+   document discovery`_), its document DOCTYPE (see list below),
+   and by the document STATUS (see list below).
+   
+   The `--skip` option may be used more than once.
+
+   DOCTYPE can be one of: 
+     Asciidoc, Docbook4XML, Docbook5XML, DocbookSGML, or Linuxdoc
+     (See also output of `--doctypes`)
+
+   STATUS can be one of: 
+     source, sources, output, outputs, published, stale, broken, new
+     orphan, orphans, orphaned, problems, work, all
+     (See also output of `--statustypes`)
+
+--resources RESOURCEDIR (default: ['images', 'resources'])
+   Some source documents provide images, scripts and other content.  These
+   files are usually stored in a directory such as ./images/ that need to be
+   copied intact into the output directory.  Adjust the set of resource
+   directories wyth this option.
+
+   The `--resources` option may be used more than once.
+
+--loglevel LOGLEVEL (default: ERROR)
+   set the loglevel to LOGLEVEL; can be passed as numeric or textual; in
+   increasing order: CRITICAL (50), ERROR (40), WARNING (30), INFO (20),
+   DEBUG (10); N.B. the text names are not case-sensitive: 'info' is OK
+
+-c, --configfile, --config-file, --cfg CONFIGFILE (default: None)
+   Specify the name of a CONFIGFILE containing parameters to be read for
+   this invocation; an INI-style configuration file.  A sample can be
+   generated with --dump-cfg.  Although only one CONFIGFILE can be specified
+   via the environment or the command-line, the system config file
+   (/etc/ldptool/ldptool.ini) is always read.
+
+--dump_cli, --dump-cli
+  Produce the resulting, merged configuration as in CLI form.  (After
+  processing all configuration sources (defaults, system configuration, user
+  configuration, environment variables, command-line.)
+
+--dump_env, --dump-env
+  Produce the resulting, merged configuration as a shell environment file.
+
+--dump_cfg, --dump-cfg
+  Produce the resulting, merged configuration as an INI-configuration file.
+
+--debug_options, --debug-options
+  Provide lots of debugging information on option-processing; see also
+  `--loglevel debug`.
+
+
+Source document discovery
+-------------------------
+Almost all documentation formats provide the possibility that a document can
+span multiple files.  Although more than half of the LDP document collection
+consists of single-file HOWTO contributions, there are a number of documents
+that are composed of dozens, even hundreds of files.  In order to accommodate
+both the simple documents and these much more complex documents, LDP adopted a
+simple (though not unique) naming strategy to allow a single document to span
+multiple files::
+
+  Each document is referred to by a stem, which is the filename
+  without any extension.  A single file document is simple
+  STEM.EXT.  A document that requires many files must be contained
+  in a directory with the STEM name.  Therefore, the primary
+  source document will always be called either STEM.EXT or
+  STEM/STEM.EXT.
+
+(If there is a STEM/STEM.xml and STEM/STEM.sgml in the same directory, that is
+an error, and `ldptool` will freak out and shoot pigeons.)
+
+During document discovery, `ldptool` will walk through all of the source
+directories specified with `--sourcedir` and build a complete list of all
+identifiable source documents.  Then, it will walk through the publication
+directory `--pubdir` and match up each output directory (by its STEM) with the
+corresponding STEM found in one of the source directories.
+
+Then, `ldptool` can then determine whether any source files are newer.  It uses
+content-hashing, i.e. MD5, and if a source file is newer, the status is
+`stale`.  If there is no matching output, the source file is `new`.  If
+there's an output with no source, that is in `orphan`.  See the
+`--statustypes` output for the full list of STATUS types.
+
+
+Examples
+--------
+To build and publish a single document::
+
+  $ ldptool --publish DocBook-Demystification-HOWTO
+  $ ldptool --publish ~/vcs/LDP/LDP/howto/docbook/Valgrind-HOWTO.xml
+
+To build and publish anything that is new or updated work::
+
+  $ ldptool --publish
+  $ ldptool --publish work
+
+To (re-)build and publish everything, regardless of state::
+
+  $ ldptool --publish all
+
+To generate a specific output (into a --builddir)::
+
+  $ ldptool --build DocBook-Demystification-HOWTO
+
+To generate all outputs into a --builddir (should exist)::
+
+  $ ldptool --builddir ~/tmp/scratch-directory/ --build all
+
+To build new/updated work, but pass over a trouble-maker::
+
+  $ ldptool --build --skip HOWTO-INDEX
+
+To loudly generate all outputs, except a trouble-maker::
+
+  $ ldptool --build all --loglevel debug --skip HOWTO-INDEX
+
+To print out a shell script for building a specific document::
+
+  $ ldptool --script TransparentProxy
+  $ ldptool --script ~/vcs/LDP/LDP/howto/docbook/Assembly-HOWTO.xml
+
+
+Environment
+-----------
+
+The `ldptool` accepts configuration via environment variables.  All such
+environment variables are prefixed with the name `LDPTOOL_`.
+
+The name of each variable is constructed from the primary
+command-line option name.  The `-b` is better known as `--builddir`, so the
+environment variable would be `LDPTOOL_BUILDDIR`.  Similarly, the environment
+variable names for each of the handlers can be derived from the name of the
+handler and its option.  For example, the Asciidoc processor needs to have
+access to the `xmllint` and `asciidoc` utilities.  
+
+The environment variable corresponding to the CLI option `--asciidoc-xmllint`
+would be `LDPTOOL_ASCIIDOC_XMLLINT`.  Similarly, `--asciidoc-asciidoc` should
+be `LDPTOOL_ASCIIDOC_ASCIIDOC`.
+
+Variables accepting multiple options use the comma as a separator::
+
+  LDPTOOL_RESOURCES=images,resources
+
+The complete listing of possible environment variables with all current values
+can be printed by using `ldptool --dump-env`.  
+
+Configuration file
+------------------
+The system-installed configuration file is `/etc/ldptool/ldptool.ini`.  The
+format is a simple INI-style configuration file with a block for the main
+program and a block for each handler.  Here's a partial example::
+
+  [ldptool] 
+  resources = images,
+          resources
+  loglevel = 40
+  
+  [ldptool-asciidoc]
+  asciidoc = /usr/bin/asciidoc
+  xmllint = /usr/bin/xmllint
+
+Note that the comma separates multiple values for a single option
+(`resources`) in the above config fragment.
+
+The complete, current configuration file can be printed by using `ldptool
+--dump-cfg`.  
+
+
+Configuration option fragments for each DOCTYPE handler
+-------------------------------------------------------
+Every source format has a single handler and each DOCTYPE handler may require
+a different set of executables and/or data files to complete its job.  The
+defaults depend on the platform and are detected at runtime.  In most cases,
+the commands are found in `/usr/bin` (see below).  The data files, for example
+the LDP XSL files and the docbook.rng, may live in different places on
+different systems.
+
+If a given DOCTYPE handler cannot find all of its requirements, it will
+complain to STDERR during execution, but will not abort the rest of the run.
+
+If, for some reason, `ldptool` cannot find data files, but you know where they
+are, consider generating a configuration file with the `--dump-cfg` option,
+adjusting the relevant options and then passing the `--configfile your.ini` to
+specify these paths.
+
+
+Asciidoc
+--------
+--asciidoc-asciidoc PATH
+  full path to asciidoc [/usr/bin/asciidoc]
+--asciidoc-xmllint PATH
+  full path to xmllint [/usr/bin/xmllint]
+
+N.B. The Asciidoc processor simply converts the source document to a
+Docbook4XML document and then uses the richer Docbook4XML toolchain.
+
+Docbook4XML
+-----------
+--docbook4xml-xslchunk PATH
+  full path to LDP HTML chunker XSL
+--docbook4xml-xslsingle PATH
+  full path to LDP HTML single-page XSL
+--docbook4xml-xslprint PATH
+  full path to LDP FO print XSL
+--docbook4xml-xmllint PATH
+  full path to xmllint [/usr/bin/xmllint]
+--docbook4xml-xsltproc PATH
+  full path to xsltproc [/usr/bin/xsltproc]
+--docbook4xml-html2text PATH
+  full path to html2text [/usr/bin/html2text]
+--docbook4xml-fop PATH
+  full path to fop [/usr/bin/fop]
+--docbook4xml-dblatex PATH
+  full path to dblatex [/usr/bin/dblatex]
+
+Docbook5XML
+-----------
+--docbook5xml-xslchunk PATH
+  full path to LDP HTML chunker XSL
+--docbook5xml-xslsingle PATH
+  full path to LDP HTML single-page XSL
+--docbook5xml-xslprint PATH
+  full path to LDP FO print XSL
+--docbook5xml-rngfile PATH
+  full path to docbook.rng
+--docbook5xml-xmllint PATH
+  full path to xmllint [/usr/bin/xmllint]
+--docbook5xml-xsltproc PATH
+  full path to xsltproc [/usr/bin/xsltproc]
+--docbook5xml-html2text PATH
+  full path to html2text [/usr/bin/html2text]
+--docbook5xml-fop PATH
+  full path to fop [/usr/bin/fop]
+--docbook5xml-dblatex PATH
+  full path to dblatex [/usr/bin/dblatex]
+--docbook5xml-jing PATH
+  full path to jing [/usr/bin/jing]
+
+DocbookSGML
+-----------
+--docbooksgml-docbookdsl PATH
+  full path to html/docbook.dsl
+--docbooksgml-ldpdsl PATH
+  full path to ldp/ldp.dsl [None]
+--docbooksgml-jw PATH
+  full path to jw [/usr/bin/jw]
+--docbooksgml-html2text PATH
+  full path to html2text [/usr/bin/html2text]
+--docbooksgml-openjade PATH
+  full path to openjade [/usr/bin/openjade]
+--docbooksgml-dblatex PATH
+  full path to dblatex [/usr/bin/dblatex]
+--docbooksgml-collateindex PATH
+  full path to collateindex
+
+Linuxdoc
+--------
+--linuxdoc-sgmlcheck PATH
+  full path to sgmlcheck [/usr/bin/sgmlcheck]
+--linuxdoc-sgml2html PATH
+  full path to sgml2html [/usr/bin/sgml2html]
+--linuxdoc-html2text PATH
+  full path to html2text [/usr/bin/html2text]
+--linuxdoc-htmldoc PATH
+  full path to htmldoc [/usr/bin/htmldoc]
+

docs/ldptool.1

@@ -0,0 +1,533 @@
+.\" Man page generated from reStructuredText.
+.
+.TH "LDPTOOL" "1" "May 16, 2016" "1.9.2" "ldptool"
+.SH NAME
+ldptool \- DocBook, Linuxdoc and Asciidoc build/publishing tool.
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.SH SYNOPSIS
+.sp
+\fBldptool\fP [\fIoptions\fP]  [\fIpathname\fP [...]]
+.SH DESCRIPTION
+.sp
+\fBldptool\fP creates chunked HTML, single\-page HTML, PDF and plain text
+outputs for each source document it is passed as a \fIpathname\fP\&.  See
+\fI\%Source document discovery\fP\&.  It will compare the source document and output
+document and rebuild an output only if the content has changed.
+.sp
+If it is not passed any arguments, \fIldptool\fP will collect all of the
+directories specified with the \-\-sourcedir option and scan through these
+directories looking for valid source documents.
+.sp
+The action taken depends on the options passed to the utility.  If no options
+are passed, then the default \fI\-\-build\fP action will be attempted.  The options
+controlling the overall program are described in the sections \fI\%Action
+options\fP and \fI\%Main options\fP\&.  All other options are relegated to the tail of
+the manpage, because they are merely configurables for individual document
+processors.
+.sp
+The \fIldptool\fP can:
+.INDENT 0.0
+.IP \(bu 2
+generate an inventory from multiple source directories (\fI\-\-sourcedir\fP)
+.IP \(bu 2
+crawl through a single output collection (\fI\-\-pubdir\fP)
+.IP \(bu 2
+match the sources to the outputs (based on document stem name)
+.IP \(bu 2
+describe the collection by type and status (\fI\-\-summary\fP)
+.IP \(bu 2
+list out individual document type and status (\fI\-\-list\fP)
+.IP \(bu 2
+describe supported source formats (\fI\-\-formats\fP)
+.IP \(bu 2
+describe the meaning of document status (\fI\-\-statustypes\fP)
+.IP \(bu 2
+build the expected (non\-configurable) set of outputs (\fI\-\-build\fP)
+.IP \(bu 2
+build and publish the outputs (\fI\-\-publish\fP)
+.IP \(bu 2
+produce runnable shell script to STDOUT (\fI\-\-script\fP)
+.IP \(bu 2
+generate configuration files that it can then take as input
+.UNINDENT
+.SH ACTION OPTIONS
+.INDENT 0.0
+.TP
+.B \-h\fP,\fB  \-\-help
+show a help message and exit
+.TP
+.B \-V\fP,\fB  \-\-version
+print out the version number
+.TP
+.B \-b\fP,\fB  \-\-build
+Build LDP documentation into the \fI\-\-builddir\fP and exit.
+This is the default action if no other action is specified.
+.TP
+.B \-p\fP,\fB  \-\-publish
+Build LDP documentation into the \fI\-\-builddir\fP\&.  If all builds are
+successful, then copy the result for each source document into the
+\fI\-\-pubdir\fP, effectively replacing (and deleting) the older documents;
+finally, remove \fI\-\-builddir\fP, if empty.
+.TP
+.B \-S\fP,\fB  \-\-script
+Print a runnable bash script to STDOUT.  This will produce a
+shell script showing what would be executed upon \fI\-\-build\fP\&.
+.TP
+.B \-l\fP,\fB  \-\-detail\fP,\fB  \-\-list
+Examine the various SOURCEDIRs and the PUBDIR and generate a report
+showing the FORMAT of the source document and STATUS of the document.
+Add the \fI\-\-verbose\fP flag for more information.
+.TP
+.B \-t\fP,\fB  \-\-summary
+Examine the various SOURCEDIRs and the PUBDIR and generate a short
+report summarizing documents by STATUS and by DOCTYPE.  Add the
+\fI\-\-verbose\fP flag for more information.
+.TP
+.B \-T\fP,\fB  \-\-doctypes\fP,\fB  \-\-formats\fP,\fB  \-\-format\fP,\fB  \-\-list\-doctypes\fP,\fB  \-\-list\-formats
+List the supported DOCTYPEs; there is one processor for each DOCTYPE.
+.TP
+.B \-\-statustypes\fP,\fB  \-\-list\-statustypes
+List the possible document STATUS types.  There are only seven basic status
+types, but several synonyms and groups of STATUS types (internally called
+\(aqclasses\(aq).
+.UNINDENT
+.SH MAIN OPTIONS
+.INDENT 0.0
+.TP
+.B \-s, \-\-sourcedir, \-\-source\-dir, \-\-source\-directory SOURCEDIR (default: None)
+Specify the name of a SOURCEDIR which contains source documents.  See
+also \fI\%Source document discovery\fP\&.
+.sp
+The \fI\-\-sourcedir\fP option may be used more than once.
+.TP
+.B \-o, \-\-pubdir, \-\-output, \-\-outputdir, \-\-outdir PUBDIR (default: None)
+Specify the the name of a PUBDIR.  Used as the publication if the build
+succeeds.  When \fI\-\-publish\fP is used with \fI\-\-pubdir\fP, the output of
+a successful document build will be used to replace any existing document
+output directory in PUBDIR.
+.TP
+.B \-d, \-\-builddir, \-\-build\-dir, \-\-build\-directory BUILDDIR (default: \(aqldptool\-build\(aq)
+Specify the name of a BUILDDIR.  A scratch directory used to build each
+source document; directory is temporary and will be removed if the
+build succeeds AND \fI\-\-publish\fP has been requested.  Under the \fI\-\-build\fP
+action, all output directories and contents remain in the BUILDDIR for
+inspection.
+.TP
+.B \-\-verbose [True | False] (default: False)
+Provide more information in \-\-list and \-\-detail actions.  The option can
+be thrown without an argument which is equivalent to True.  To allow the
+CLI to supersede environment or configuration file values, \fI\-\-verbose
+false\fP is also supported.
+.TP
+.B \-\-skip [STEM | DOCTYPE | STATUS]
+Specify a source document name, document type or document status to skip
+during processing.  Each document is known by its STEM (see also \fI\%Source
+document discovery\fP), its document DOCTYPE (see list below),
+and by the document STATUS (see list below).
+.sp
+The \fI\-\-skip\fP option may be used more than once.
+.INDENT 7.0
+.TP
+.B DOCTYPE can be one of:
+Asciidoc, Docbook4XML, Docbook5XML, DocbookSGML, or Linuxdoc
+(See also output of \fI\-\-doctypes\fP)
+.TP
+.B STATUS can be one of:
+source, sources, output, outputs, published, stale, broken, new
+orphan, orphans, orphaned, problems, work, all
+(See also output of \fI\-\-statustypes\fP)
+.UNINDENT
+.TP
+.B \-\-resources RESOURCEDIR (default: [\(aqimages\(aq, \(aqresources\(aq])
+Some source documents provide images, scripts and other content.  These
+files are usually stored in a directory such as ./images/ that need to be
+copied intact into the output directory.  Adjust the set of resource
+directories wyth this option.
+.sp
+The \fI\-\-resources\fP option may be used more than once.
+.TP
+.B \-\-loglevel LOGLEVEL (default: ERROR)
+set the loglevel to LOGLEVEL; can be passed as numeric or textual; in
+increasing order: CRITICAL (50), ERROR (40), WARNING (30), INFO (20),
+DEBUG (10); N.B. the text names are not case\-sensitive: \(aqinfo\(aq is OK
+.TP
+.B \-c, \-\-configfile, \-\-config\-file, \-\-cfg CONFIGFILE (default: None)
+Specify the name of a CONFIGFILE containing parameters to be read for
+this invocation; an INI\-style configuration file.  A sample can be
+generated with \-\-dump\-cfg.  Although only one CONFIGFILE can be specified
+via the environment or the command\-line, the system config file
+(/etc/ldptool/ldptool.ini) is always read.
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-dump_cli\fP,\fB  \-\-dump\-cli
+Produce the resulting, merged configuration as in CLI form.  (After
+processing all configuration sources (defaults, system configuration, user
+configuration, environment variables, command\-line.)
+.TP
+.B \-\-dump_env\fP,\fB  \-\-dump\-env
+Produce the resulting, merged configuration as a shell environment file.
+.TP
+.B \-\-dump_cfg\fP,\fB  \-\-dump\-cfg
+Produce the resulting, merged configuration as an INI\-configuration file.
+.TP
+.B \-\-debug_options\fP,\fB  \-\-debug\-options
+Provide lots of debugging information on option\-processing; see also
+\fI\-\-loglevel debug\fP\&.
+.UNINDENT
+.SH SOURCE DOCUMENT DISCOVERY
+.sp
+Almost all documentation formats provide the possibility that a document can
+span multiple files.  Although more than half of the LDP document collection
+consists of single\-file HOWTO contributions, there are a number of documents
+that are composed of dozens, even hundreds of files.  In order to accommodate
+both the simple documents and these much more complex documents, LDP adopted a
+simple (though not unique) naming strategy to allow a single document to span
+multiple files:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+Each document is referred to by a stem, which is the filename
+without any extension.  A single file document is simple
+STEM.EXT.  A document that requires many files must be contained
+in a directory with the STEM name.  Therefore, the primary
+source document will always be called either STEM.EXT or
+STEM/STEM.EXT.
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+(If there is a STEM/STEM.xml and STEM/STEM.sgml in the same directory, that is
+an error, and \fIldptool\fP will freak out and shoot pigeons.)
+.sp
+During document discovery, \fIldptool\fP will walk through all of the source
+directories specified with \fI\-\-sourcedir\fP and build a complete list of all
+identifiable source documents.  Then, it will walk through the publication
+directory \fI\-\-pubdir\fP and match up each output directory (by its STEM) with the
+corresponding STEM found in one of the source directories.
+.sp
+Then, \fIldptool\fP can then determine whether any source files are newer.  It uses
+content\-hashing, i.e. MD5, and if a source file is newer, the status is
+\fIstale\fP\&.  If there is no matching output, the source file is \fInew\fP\&.  If
+there\(aqs an output with no source, that is in \fIorphan\fP\&.  See the
+\fI\-\-statustypes\fP output for the full list of STATUS types.
+.SH EXAMPLES
+.sp
+To build and publish a single document:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+$ ldptool \-\-publish DocBook\-Demystification\-HOWTO
+$ ldptool \-\-publish ~/vcs/LDP/LDP/howto/docbook/Valgrind\-HOWTO.xml
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+To build and publish anything that is new or updated work:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+$ ldptool \-\-publish
+$ ldptool \-\-publish work
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+To (re\-)build and publish everything, regardless of state:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+$ ldptool \-\-publish all
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+To generate a specific output (into a \-\-builddir):
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+$ ldptool \-\-build DocBook\-Demystification\-HOWTO
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+To generate all outputs into a \-\-builddir (should exist):
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+$ ldptool \-\-builddir ~/tmp/scratch\-directory/ \-\-build all
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+To build new/updated work, but pass over a trouble\-maker:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+$ ldptool \-\-build \-\-skip HOWTO\-INDEX
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+To loudly generate all outputs, except a trouble\-maker:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+$ ldptool \-\-build all \-\-loglevel debug \-\-skip HOWTO\-INDEX
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+To print out a shell script for building a specific document:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+$ ldptool \-\-script TransparentProxy
+$ ldptool \-\-script ~/vcs/LDP/LDP/howto/docbook/Assembly\-HOWTO.xml
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.SH ENVIRONMENT
+.sp
+The \fIldptool\fP accepts configuration via environment variables.  All such
+environment variables are prefixed with the name \fILDPTOOL_\fP\&.
+.sp
+The name of each variable is constructed from the primary
+command\-line option name.  The \fI\-b\fP is better known as \fI\-\-builddir\fP, so the
+environment variable would be \fILDPTOOL_BUILDDIR\fP\&.  Similarly, the environment
+variable names for each of the handlers can be derived from the name of the
+handler and its option.  For example, the Asciidoc processor needs to have
+access to the \fIxmllint\fP and \fIasciidoc\fP utilities.
+.sp
+The environment variable corresponding to the CLI option \fI\-\-asciidoc\-xmllint\fP
+would be \fILDPTOOL_ASCIIDOC_XMLLINT\fP\&.  Similarly, \fI\-\-asciidoc\-asciidoc\fP should
+be \fILDPTOOL_ASCIIDOC_ASCIIDOC\fP\&.
+.sp
+Variables accepting multiple options use the comma as a separator:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+LDPTOOL_RESOURCES=images,resources
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+The complete listing of possible environment variables with all current values
+can be printed by using \fIldptool \-\-dump\-env\fP\&.
+.SH CONFIGURATION FILE
+.sp
+The system\-installed configuration file is \fI/etc/ldptool/ldptool.ini\fP\&.  The
+format is a simple INI\-style configuration file with a block for the main
+program and a block for each handler.  Here\(aqs a partial example:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+[ldptool]
+resources = images,
+        resources
+loglevel = 40
+
+[ldptool\-asciidoc]
+asciidoc = /usr/bin/asciidoc
+xmllint = /usr/bin/xmllint
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+Note that the comma separates multiple values for a single option
+(\fIresources\fP) in the above config fragment.
+.sp
+The complete, current configuration file can be printed by using \fIldptool
+\-\-dump\-cfg\fP\&.
+.SH CONFIGURATION OPTION FRAGMENTS FOR EACH DOCTYPE HANDLER
+.sp
+Every source format has a single handler and each DOCTYPE handler may require
+a different set of executables and/or data files to complete its job.  The
+defaults depend on the platform and are detected at runtime.  In most cases,
+the commands are found in \fI/usr/bin\fP (see below).  The data files, for example
+the LDP XSL files and the docbook.rng, may live in different places on
+different systems.
+.sp
+If a given DOCTYPE handler cannot find all of its requirements, it will
+complain to STDERR during execution, but will not abort the rest of the run.
+.sp
+If, for some reason, \fIldptool\fP cannot find data files, but you know where they
+are, consider generating a configuration file with the \fI\-\-dump\-cfg\fP option,
+adjusting the relevant options and then passing the \fI\-\-configfile your.ini\fP to
+specify these paths.
+.SH ASCIIDOC
+.INDENT 0.0
+.TP
+.BI \-\-asciidoc\-asciidoc \ PATH
+full path to asciidoc [/usr/bin/asciidoc]
+.TP
+.BI \-\-asciidoc\-xmllint \ PATH
+full path to xmllint [/usr/bin/xmllint]
+.UNINDENT
+.sp
+N.B. The Asciidoc processor simply converts the source document to a
+Docbook4XML document and then uses the richer Docbook4XML toolchain.
+.SH DOCBOOK4XML
+.INDENT 0.0
+.TP
+.BI \-\-docbook4xml\-xslchunk \ PATH
+full path to LDP HTML chunker XSL
+.TP
+.BI \-\-docbook4xml\-xslsingle \ PATH
+full path to LDP HTML single\-page XSL
+.TP
+.BI \-\-docbook4xml\-xslprint \ PATH
+full path to LDP FO print XSL
+.TP
+.BI \-\-docbook4xml\-xmllint \ PATH
+full path to xmllint [/usr/bin/xmllint]
+.TP
+.BI \-\-docbook4xml\-xsltproc \ PATH
+full path to xsltproc [/usr/bin/xsltproc]
+.TP
+.BI \-\-docbook4xml\-html2text \ PATH
+full path to html2text [/usr/bin/html2text]
+.TP
+.BI \-\-docbook4xml\-fop \ PATH
+full path to fop [/usr/bin/fop]
+.TP
+.BI \-\-docbook4xml\-dblatex \ PATH
+full path to dblatex [/usr/bin/dblatex]
+.UNINDENT
+.SH DOCBOOK5XML
+.INDENT 0.0
+.TP
+.BI \-\-docbook5xml\-xslchunk \ PATH
+full path to LDP HTML chunker XSL
+.TP
+.BI \-\-docbook5xml\-xslsingle \ PATH
+full path to LDP HTML single\-page XSL
+.TP
+.BI \-\-docbook5xml\-xslprint \ PATH
+full path to LDP FO print XSL
+.TP
+.BI \-\-docbook5xml\-rngfile \ PATH
+full path to docbook.rng
+.TP
+.BI \-\-docbook5xml\-xmllint \ PATH
+full path to xmllint [/usr/bin/xmllint]
+.TP
+.BI \-\-docbook5xml\-xsltproc \ PATH
+full path to xsltproc [/usr/bin/xsltproc]
+.TP
+.BI \-\-docbook5xml\-html2text \ PATH
+full path to html2text [/usr/bin/html2text]
+.TP
+.BI \-\-docbook5xml\-fop \ PATH
+full path to fop [/usr/bin/fop]
+.TP
+.BI \-\-docbook5xml\-dblatex \ PATH
+full path to dblatex [/usr/bin/dblatex]
+.TP
+.BI \-\-docbook5xml\-jing \ PATH
+full path to jing [/usr/bin/jing]
+.UNINDENT
+.SH DOCBOOKSGML
+.INDENT 0.0
+.TP
+.BI \-\-docbooksgml\-docbookdsl \ PATH
+full path to html/docbook.dsl
+.TP
+.BI \-\-docbooksgml\-ldpdsl \ PATH
+full path to ldp/ldp.dsl [None]
+.TP
+.BI \-\-docbooksgml\-jw \ PATH
+full path to jw [/usr/bin/jw]
+.TP
+.BI \-\-docbooksgml\-html2text \ PATH
+full path to html2text [/usr/bin/html2text]
+.TP
+.BI \-\-docbooksgml\-openjade \ PATH
+full path to openjade [/usr/bin/openjade]
+.TP
+.BI \-\-docbooksgml\-dblatex \ PATH
+full path to dblatex [/usr/bin/dblatex]
+.TP
+.BI \-\-docbooksgml\-collateindex \ PATH
+full path to collateindex
+.UNINDENT
+.SH LINUXDOC
+.INDENT 0.0
+.TP
+.BI \-\-linuxdoc\-sgmlcheck \ PATH
+full path to sgmlcheck [/usr/bin/sgmlcheck]
+.TP
+.BI \-\-linuxdoc\-sgml2html \ PATH
+full path to sgml2html [/usr/bin/sgml2html]
+.TP
+.BI \-\-linuxdoc\-html2text \ PATH
+full path to html2text [/usr/bin/html2text]
+.TP
+.BI \-\-linuxdoc\-htmldoc \ PATH
+full path to htmldoc [/usr/bin/htmldoc]
+.UNINDENT
+.SH AUTHOR
+Martin A. Brown <martin@linux-ip.net>
+.SH COPYRIGHT
+Manual page (C) 2016, Linux Documentation Project
+.\" Generated by docutils manpage writer.
+.

etc/ldptool.ini

@@ -56,63 +56,72 @@ loglevel = ERROR
 #
 verbose = False
 
-# -- the four main actions, probably ought not to be set in the config file
+# -- These are the main actions and they are mutually exclusive.  Pick any
+#    of them that you would like:
 #
+# publish = False
+# build = False
+# script = False
 # detail = False
 # summary = False
-# script = False
-# build = False
+# doctypes = False
+# statustypes = False
+#
 
 # -- Each of the document types may require different executables and/or data
-#    files to support processing of a specific document type.  The below
+#    files to support processing of the specific document type.  The below
 #    configuration file section fragments allow each document type processor
 #    to keep its own configurables separate from other document processors.
 #
 # -- The ldptool code uses $PATH (from the environment) to locate the
 #    executables, by default.  If the utilities are not installed in the
 #    system path, then it is possible to configure the full path to each
-#    executable, here, in this system-wide configuration file.
+#    executable in your own configuration file or in a system-wide
+#    configuration file (/etc/ldptool/ldptool.ini).
 #
-# -- Also, the data files, for example, the DocBook DSSSL and DocBook XSL
-#    stylesheets may be in a location that ldptool cannot find.  If so, it
-#    will skip building any document type if it is lacking the appropriate
-#    data files.  It is possible to configure the full path to the data files
-#    here, in this system-wide configuration file.
+# -- If specific data files are not discoverable, e.g. the DocBook DSSSL and
+#    DocBook XSL stylesheets, the ldptool will skip processing that document
+#    type.
 #
 
+[ldptool-linuxdoc]
+# htmldoc = /usr/bin/htmldoc
+# html2text = /usr/bin/html2text
+# sgml2html = /usr/bin/sgml2html
+# sgmlcheck = /usr/bin/sgmlcheck
+
+[ldptool-docbooksgml]
+# collateindex = /home/mabrown/bin/collateindex.pl
+# dblatex = /usr/bin/dblatex
+# docbookdsl = /usr/share/sgml/docbook/dsssl-stylesheets/html/docbook.dsl
+# html2text = /usr/bin/html2text
+# jw = /usr/bin/jw
+# ldpdsl = /usr/share/sgml/docbook/stylesheet/dsssl/ldp/ldp.dsl
+# openjade = /usr/bin/openjade
+
 [ldptool-docbook4xml]
-# xslchunk = /usr/share/xml/docbook/stylesheet/ldp/html/tldp-sections.xsl
 # fop = /usr/bin/fop
 # dblatex = /usr/bin/dblatex
-# xsltproc = /usr/bin/xsltproc
 # html2text = /usr/bin/html2text
+# xsltproc = /usr/bin/xsltproc
+# xslchunk = /usr/share/xml/docbook/stylesheet/ldp/html/tldp-sections.xsl
 # xslprint = /usr/share/xml/docbook/stylesheet/ldp/fo/tldp-print.xsl
 # xslsingle = /usr/share/xml/docbook/stylesheet/ldp/html/tldp-one-page.xsl
 
-[ldptool-linuxdoc]
-# sgml2html = /usr/bin/sgml2html
-# htmldoc = /usr/bin/htmldoc
-# html2text = /usr/bin/html2text
-
-[ldptool-docbooksgml]
-# ldpdsl = /usr/share/sgml/docbook/stylesheet/dsssl/ldp/ldp.dsl
-# jw = /usr/bin/jw
-# dblatex = /usr/bin/dblatex
-# html2text = /usr/bin/html2text
-# collateindex = /home/mabrown/bin/collateindex.pl
-# docbookdsl = /usr/share/sgml/docbook/dsssl-stylesheets/html/docbook.dsl
-# openjade = /usr/bin/openjade
-
 [ldptool-docbook5xml]
-# xsltproc = /usr/bin/xsltproc
 # dblatex = /usr/bin/dblatex
-# xslprint = /usr/share/xml/docbook/stylesheet/docbook-xsl-ns/fo/docbook.xsl
-# xmllint = /usr/bin/xmllint
-# xslsingle = /usr/share/xml/docbook/stylesheet/docbook-xsl-ns/html/docbook.xsl
-# xslchunk = /usr/share/xml/docbook/stylesheet/docbook-xsl-ns/html/chunk.xsl
-# rngfile = /usr/share/xml/docbook/schema/rng/5.0/docbook.rng
 # fop = /usr/bin/fop
 # jing = /usr/bin/jing
 # html2text = /usr/bin/html2text
+# rngfile = /usr/share/xml/docbook/schema/rng/5.0/docbook.rng
+# xmllint = /usr/bin/xmllint
+# xslchunk = /usr/share/xml/docbook/stylesheet/docbook-xsl-ns/html/chunk.xsl
+# xslprint = /usr/share/xml/docbook/stylesheet/docbook-xsl-ns/fo/docbook.xsl
+# xslsingle = /usr/share/xml/docbook/stylesheet/docbook-xsl-ns/html/docbook.xsl
+# xsltproc = /usr/bin/xsltproc
+
+[ldptool-asciidoc]
+# asciidoc = /usr/bin/asciidoc
+# xmllint = /usr/bin/xmllint
 
 # -- end of file

setup.py

@@ -3,6 +3,7 @@ import os
 
 import glob
 from setuptools import setup
+from tldp import VERSION
 
 
 with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as r_file:
@@ -11,25 +12,25 @@ with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as r_file:
 
 setup(
     name='tldp',
-    version='0.6.6',
+    version=VERSION,
     license='MIT',
     author='Martin A. Brown',
     author_email='martin@linux-ip.net',
     url="http://en.tldp.org/",
-    description='tools for processing all TLDP source documents',
+    description='automatic publishing tool for DocBook, Linuxdoc and Asciidoc',
     long_description=readme,
     packages=['tldp', 'tldp/doctypes'],
     test_suite='nose.collector',
     install_requires=['networkx', 'nose'],
-    include_package_data = True,
-    package_data = {'extras': ['extras/collateindex.pl'],
-                    'extras/xsl': glob.glob('extras/xsl/*.xsl'),
-                    'extras/css': glob.glob('extras/css/*.css'),
-                    'extras/dsssl': glob.glob('extras/dsssl/*.dsl'),
-                    },
-    data_files = [('/etc/ldptool', ['etc/ldptool.ini']), ],
-    entry_points = {
-        'console_scripts': ['ldptool = tldp.driver:main',],
+    include_package_data=True,
+    package_data={'extras': ['extras/collateindex.pl'],
+                  'extras/xsl': glob.glob('extras/xsl/*.xsl'),
+                  'extras/css': glob.glob('extras/css/*.css'),
+                  'extras/dsssl': glob.glob('extras/dsssl/*.dsl'),
+                  },
+    data_files=[('/etc/ldptool', ['etc/ldptool.ini']), ],
+    entry_points={
+        'console_scripts': ['ldptool = tldp.driver:main', ],
     },
     classifiers=[
         'Development Status :: 4 - Beta',

tests/example.py

@@ -1,5 +1,9 @@
+# -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
 
 import os
 import codecs
@@ -14,71 +18,62 @@ opd = os.path.dirname
 opa = os.path.abspath
 sampledocs = opa(opj(opd(__file__), 'sample-documents'))
 
-ex_linuxdoc = Namespace(
-               doctype=tldp.doctypes.linuxdoc.Linuxdoc,
-               filename=opj(sampledocs, 'linuxdoc-simple.sgml'),
-              )
 
-ex_docbooksgml = Namespace(
-                  doctype=tldp.doctypes.docbooksgml.DocbookSGML,
-                  filename=opj(sampledocs, 'docbooksgml-simple.sgml'),
-                 )
-
-ex_docbook4xml = Namespace(
-                 doctype=tldp.doctypes.docbook4xml.Docbook4XML,
-                 filename=opj(sampledocs, 'docbook4xml-simple.xml'),
-                )
-
-ex_docbook5xml = Namespace(
-                 doctype=tldp.doctypes.docbook5xml.Docbook5XML,
-                 filename=opj(sampledocs, 'docbook5xml-simple.xml'),
-                )
-
-ex_asciidoc = Namespace(
-           doctype=tldp.doctypes.asciidoc.Asciidoc,
-           filename=opj(sampledocs, 'asciidoc-complete.txt'),
-          )
-#
-# ex_rst = Namespace(
-#           doctype=tldp.doctypes.rst.RestructuredText,
-#           filename=opj(sampledocs, 'restructuredtext-simple.rst'),
-#          )
-#
-# ex_markdown = Namespace(
-#                doctype=tldp.doctypes.markdown.Markdown,
-#                filename=opj(sampledocs, 'markdown-simple.md'),
-#               )
-
-ex_linuxdoc_dir = Namespace(
-               doctype=tldp.doctypes.linuxdoc.Linuxdoc,
-               filename=opj(sampledocs, 'Linuxdoc-Larger',
-                            'Linuxdoc-Larger.sgml'),
-              )
-
-ex_docbook4xml_dir = Namespace(
-               doctype=tldp.doctypes.docbook4xml.Docbook4XML,
-               filename=opj(sampledocs, 'DocBook-4.2-WHYNOT',
-                            'DocBook-4.2-WHYNOT.xml'),
-              )
-
-ex_docbooksgml_dir = Namespace(
-               doctype=tldp.doctypes.docbooksgml.DocbookSGML,
-               filename=opj(sampledocs, 'DocBookSGML-Larger',
-                            'DocBookSGML-Larger.sgml'),
-              )
-
-unknown_doctype = Namespace(
-               doctype=None,
-               filename=opj(sampledocs, 'Unknown-Doctype.xqf'),
-              )
-
-# -- a bit ugly, but grab each dict
-sources = [y for x, y in locals().items() if x.startswith('ex_')]
-
-for ex in sources:
+def load_content(ex):
     with codecs.open(ex.filename, encoding='utf-8') as f:
         ex.content = f.read()
     ex.stem, ex.ext = stem_and_ext(ex.filename)
 
 
+ex_linuxdoc = Namespace(
+    doctype=tldp.doctypes.linuxdoc.Linuxdoc,
+    filename=opj(sampledocs, 'linuxdoc-simple.sgml'),)
+
+ex_docbooksgml = Namespace(
+    doctype=tldp.doctypes.docbooksgml.DocbookSGML,
+    filename=opj(sampledocs, 'docbooksgml-simple.sgml'),)
+
+ex_docbook4xml = Namespace(
+    doctype=tldp.doctypes.docbook4xml.Docbook4XML,
+    filename=opj(sampledocs, 'docbook4xml-simple.xml'),)
+
+ex_docbook5xml = Namespace(
+    doctype=tldp.doctypes.docbook5xml.Docbook5XML,
+    filename=opj(sampledocs, 'docbook5xml-simple.xml'),)
+
+ex_asciidoc = Namespace(
+    doctype=tldp.doctypes.asciidoc.Asciidoc,
+    filename=opj(sampledocs, 'asciidoc-complete.txt'),)
+
+ex_linuxdoc_dir = Namespace(
+    doctype=tldp.doctypes.linuxdoc.Linuxdoc,
+    filename=opj(sampledocs, 'Linuxdoc-Larger',
+                             'Linuxdoc-Larger.sgml'),)
+
+ex_docbook4xml_dir = Namespace(
+    doctype=tldp.doctypes.docbook4xml.Docbook4XML,
+    filename=opj(sampledocs, 'DocBook-4.2-WHYNOT',
+                             'DocBook-4.2-WHYNOT.xml'),)
+
+ex_docbooksgml_dir = Namespace(
+    doctype=tldp.doctypes.docbooksgml.DocbookSGML,
+    filename=opj(sampledocs, 'DocBookSGML-Larger',
+                             'DocBookSGML-Larger.sgml'),)
+
+# -- a bit ugly, but grab each dict
+sources = [y for x, y in locals().items() if x.startswith('ex_')]
+
+for ex in sources:
+    load_content(ex)
+
+unknown_doctype = Namespace(
+    doctype=None,
+    filename=opj(sampledocs, 'Unknown-Doctype.xqf'),)
+
+broken_docbook4xml = Namespace(
+    doctype=tldp.doctypes.docbook4xml.Docbook4XML,
+    filename=opj(sampledocs, 'docbook4xml-broken.xml'),)
+
+load_content(broken_docbook4xml)
+
 # -- end of file

tests/long_driver.py

@@ -1,3 +1,6 @@
+# -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 
@@ -14,6 +17,24 @@ import example
 import tldp.driver
 
 
+class TestDriverRun(TestInventoryBase):
+
+    def test_run_status_selection(self):
+        self.add_docbook4xml_xsl_to_config()
+        c = self.config
+        self.add_stale('Asciidoc-Stale-HOWTO', example.ex_asciidoc)
+        self.add_new('DocBook4XML-New-HOWTO', example.ex_docbook4xml)
+        argv = self.argv
+        argv.extend(['--publish', 'stale'])
+        argv.extend(['--docbook4xml-xslprint', c.docbook4xml_xslprint])
+        argv.extend(['--docbook4xml-xslchunk', c.docbook4xml_xslchunk])
+        argv.extend(['--docbook4xml-xslsingle', c.docbook4xml_xslsingle])
+        exitcode = tldp.driver.run(argv)
+        self.assertEqual(exitcode, os.EX_OK)
+        inv = tldp.inventory.Inventory(c.pubdir, c.sourcedir)
+        self.assertEqual(1, len(inv.published.keys()))
+
+
 class TestDriverBuild(TestInventoryBase):
 
     def test_build_one_broken(self):
@@ -46,6 +67,17 @@ class TestDriverBuild(TestInventoryBase):
 
 class TestDriverPublish(TestInventoryBase):
 
+    def test_publish_fail_because_broken(self):
+        c = self.config
+        c.publish = True
+        self.add_new('Frobnitz-DocBook-XML-4-HOWTO', example.ex_docbook4xml)
+        self.add_stale('Broken-DocBook-XML-4-HOWTO', example.broken_docbook4xml)
+        inv = tldp.inventory.Inventory(c.pubdir, c.sourcedir)
+        self.assertEqual(2, len(inv.all.keys()))
+        docs = inv.all.values()
+        exitcode = tldp.driver.publish(c, docs)
+        self.assertNotEqual(exitcode, os.EX_OK)
+
     def test_publish_docbook5xml(self):
         c = self.config
         c.publish = True
@@ -54,7 +86,7 @@ class TestDriverPublish(TestInventoryBase):
         self.assertEqual(1, len(inv.all.keys()))
         docs = inv.all.values()
         exitcode = tldp.driver.publish(c, docs)
-        self.assertEqual(exitcode, 0)
+        self.assertEqual(exitcode, os.EX_OK)
         doc = docs.pop(0)
         self.assertTrue(doc.output.iscomplete)
 
@@ -67,7 +99,7 @@ class TestDriverPublish(TestInventoryBase):
         self.assertEqual(1, len(inv.all.keys()))
         docs = inv.all.values()
         exitcode = tldp.driver.publish(c, docs)
-        self.assertEqual(exitcode, 0)
+        self.assertEqual(exitcode, os.EX_OK)
         doc = docs.pop(0)
         self.assertTrue(doc.output.iscomplete)
 
@@ -81,7 +113,7 @@ class TestDriverPublish(TestInventoryBase):
         docs = inv.all.values()
         c.skip = []
         exitcode = tldp.driver.publish(c, docs)
-        self.assertEqual(exitcode, 0)
+        self.assertEqual(exitcode, os.EX_OK)
         doc = docs.pop(0)
         self.assertTrue(doc.output.iscomplete)
 
@@ -94,7 +126,7 @@ class TestDriverPublish(TestInventoryBase):
         docs = inv.all.values()
         c.skip = []
         exitcode = tldp.driver.publish(c, docs)
-        self.assertEqual(exitcode, 0)
+        self.assertEqual(exitcode, os.EX_OK)
         doc = docs.pop(0)
         self.assertTrue(doc.output.iscomplete)
 
@@ -107,7 +139,7 @@ class TestDriverPublish(TestInventoryBase):
         self.assertEqual(1, len(inv.all.keys()))
         docs = inv.all.values()
         exitcode = tldp.driver.publish(c, docs)
-        self.assertEqual(exitcode, 0)
+        self.assertEqual(exitcode, os.EX_OK)
         doc = docs.pop(0)
         self.assertTrue(doc.output.iscomplete)
 
@@ -117,7 +149,7 @@ class TestDriverPublish(TestInventoryBase):
         c.publish = True
         doc = SourceDocument(example.ex_docbooksgml_dir.filename)
         exitcode = tldp.driver.publish(c, [doc])
-        self.assertEqual(exitcode, 0)
+        self.assertEqual(exitcode, os.EX_OK)
         self.assertTrue(doc.output.iscomplete)
         outputimages = os.path.join(doc.output.dirname, 'images')
         self.assertTrue(os.path.exists(outputimages))

tests/long_inventory.py

@@ -0,0 +1,93 @@
+# -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
+
+from __future__ import absolute_import, division, print_function
+
+import io
+import os
+import codecs
+import shutil
+
+from tldptesttools import TestInventoryBase, TestSourceDocSkeleton
+
+# -- Test Data
+import example
+
+# -- SUT
+import tldp.driver
+
+opb = os.path.basename
+opj = os.path.join
+
+
+class TestInventoryHandling(TestInventoryBase):
+
+    def test_lifecycle(self):
+        self.add_docbook4xml_xsl_to_config()
+        c = self.config
+        argv = self.argv
+        argv.extend(['--publish'])
+        argv.extend(['--docbook4xml-xslprint', c.docbook4xml_xslprint])
+        argv.extend(['--docbook4xml-xslchunk', c.docbook4xml_xslchunk])
+        argv.extend(['--docbook4xml-xslsingle', c.docbook4xml_xslsingle])
+        mysource = TestSourceDocSkeleton(c.sourcedir)
+        ex = example.ex_docbook4xml_dir
+        exdir = os.path.dirname(ex.filename)
+        mysource.copytree(exdir)
+        inv = tldp.inventory.Inventory(c.pubdir, c.sourcedir)
+        self.assertEqual(1, len(inv.new.keys()))
+
+        # -- run first build (will generate MD5SUMS file
+        #
+        exitcode = tldp.driver.run(argv)
+        self.assertEqual(exitcode, os.EX_OK)
+        inv = tldp.inventory.Inventory(c.pubdir, c.sourcedir)
+        self.assertEqual(1, len(inv.published.keys()))
+
+        # -- remove the generated MD5SUMS file, ensure rebuild occurs
+        #
+        doc = inv.published.values().pop()
+        os.unlink(doc.output.MD5SUMS)
+        self.assertEqual(dict(), doc.output.md5sums)
+        inv = tldp.inventory.Inventory(c.pubdir, c.sourcedir)
+        self.assertEqual(1, len(inv.stale.keys()))
+        if not os.path.isdir(c.builddir):
+            os.mkdir(c.builddir)
+        exitcode = tldp.driver.run(argv)
+        inv = tldp.inventory.Inventory(c.pubdir, c.sourcedir)
+        self.assertEqual(1, len(inv.published.keys()))
+
+        # -- remove a source file, add a source file, change a source file
+        #
+        main = opj(mysource.dirname, opb(exdir), opb(ex.filename))
+        disappearing = opj(mysource.dirname, opb(exdir), 'disappearing.xml')
+        brandnew = opj(mysource.dirname, opb(exdir), 'brandnew.xml')
+        shutil.copy(disappearing, brandnew)
+        os.unlink(opj(mysource.dirname, opb(exdir), 'disappearing.xml'))
+        with codecs.open(main, 'w', encoding='utf-8') as f:
+            f.write(ex.content.replace('FIXME', 'TOTALLY-FIXED'))
+        inv = tldp.inventory.Inventory(c.pubdir, c.sourcedir)
+        self.assertEqual(1, len(inv.stale.keys()))
+        stdout = io.StringIO()
+        c.verbose = True
+        tldp.driver.detail(c, inv.all.values(), file=stdout)
+        stdout.seek(0)
+        data = stdout.read()
+        self.assertTrue('new source' in data)
+        self.assertTrue('gone source' in data)
+        self.assertTrue('changed source' in data)
+
+        # -- rebuild (why not?)
+        #
+        if not os.path.isdir(c.builddir):
+            os.mkdir(c.builddir)
+        exitcode = tldp.driver.run(argv)
+        self.assertEqual(exitcode, os.EX_OK)
+        inv = tldp.inventory.Inventory(c.pubdir, c.sourcedir)
+        self.assertEqual(1, len(inv.published.keys()))
+
+        # -- remove a file (known extraneous file, build should succeed)
+
+#
+# -- end of file

tests/sample-documents/Bad-Dir-Multiple-Doctypes/Bad-Dir-Multiple-Doctypes.sgml

@@ -1,7 +1,7 @@
 <!DOCTYPE article PUBLIC "-//OASIS//DTD DocBook V4.1//EN">
 <article>
   <articleinfo>
-    <title>T</title>
+    <title>Bad Dir Multiple Doctypes (DocBook SGML 4.1)</title>
     <author>
       <firstname>A</firstname> <surname>B</surname>
       <affiliation>

tests/sample-documents/Bad-Dir-Multiple-Doctypes/Bad-Dir-Multiple-Doctypes.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <article xmlns="http://docbook.org/ns/docbook" version="5.0" xml:lang="en">
-  <title>Simple article</title>
+  <title>Bad Dir Multiple Doctypes (DocBook XML 5.0)</title>
   <para>This is a ridiculously terse article.</para>
 </article>

tests/sample-documents/DocBook-4.2-WHYNOT/DocBook-4.2-WHYNOT.xml

@@ -21,6 +21,18 @@
     <sect2>
       <title>Intro</title>
       <para>Text</para>
+      <mediaobject>
+          <imageobject>
+              <imagedata fileref="images/warning.svg" format="SVG"/>
+          </imageobject>
+          <imageobject>
+              <imagedata fileref="images/warning.png" format="PNG"/>
+          </imageobject>
+          <imageobject>
+              <imagedata fileref="images/warning.jpg" format="JPG"/>
+          </imageobject>
+      </mediaobject>
+      <para>FIXME</para>
     </sect2>
   </sect1>
 

tests/sample-documents/DocBook-4.2-WHYNOT/disappearing.xml

@@ -0,0 +1,5 @@
+<section>
+  <para>
+    I am just a disappearing file, for use in the long_inventory.py test.
+  </para>
+</section>

tests/sample-documents/DocBook-4.2-WHYNOT/images/warning.svg

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+  "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.1" id="Layer_1"
+     xmlns="http://www.w3.org/2000/svg"
+     xmlns:xlink="http://www.w3.org/1999/xlink" 
+     x="0px" y="0px"
+     width="24px" 
+     height="24px" 
+     viewBox="0 0 24 24" 
+     enable-background="new 0 0 24 24"
+     xml:space="preserve">
+  <image id="image0" width="24" height="24" x="0" y="0"
+    xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAMAAADXqc3KAAAABGdBTUEAALGPC/xhBQAAACBjSFJN
+AAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAAolBMVEX////e3t7W1tbv7+/G
+xsacUlK1KSmtMTGtlJS1tbWlSkrvCAj3AADGGBilOTmlhIScWlqlpaW1AACUAADWAAC9AAApAAAQ
+AABaAAC1paWtAAAYAABCAADeAACUe3vGAAAhAAAIAABSAAAxAABzAADnAABKAACMAAClAABrAACE
+AAB7AADOKSmllJS1nJzWKSmce3vOAADGEBDnISH39/e9ra2Ltr4AAAAAAWJLR0QhxGwNFgAAANVJ
+REFUKM9tkukSgjAMhFtAAVvEA4knIgre4vn+ryZjGxtmuv1T9usmU1LGlLiDchmV1+n6avkBJWFP
+SFRkiOv1jd+QGIk3oH5DhoqEeH40TmjGw/qTFKbC9OEdrDMGSBOsNmOOj6XnC1iu9F5kzOkiWOeb
+QhDwT6y2sJO2RFnBXtoSsoCDNSGPi7k9MTkl9h7ngia4uQeYe1yuzL1F+mOUQ1WqbX3njJLTQ/sx
+/40jeLb+unxlXM3jHbZInZkRBhHxY25m3vQRpVqXjNNX8v7cta7a/wIS1x0MRP1EswAAACV0RVh0
+ZGF0ZTpjcmVhdGUAMjAxNi0wNC0xOVQyMToxMToxNy0wNzowMOsKgVIAAAAldEVYdGRhdGU6bW9k
+aWZ5ADIwMTYtMDQtMTlUMjE6MTE6MTctMDc6MDCaVznuAAAAKnRFWHRTaWduYXR1cmUAYzQyYjdk
+MmQ1NjRhYWI1ODg4OTE5Nzk3MDNmMDJiNDVPEd+TAAAAQ3RFWHRTb2Z0d2FyZQBAKCMpSW1hZ2VN
+YWdpY2sgNC4yLjggOTkvMDgvMDEgY3Jpc3R5QG15c3RpYy5lcy5kdXBvbnQuY29tkbohuAAAAABJ
+RU5ErkJggg==" />
+</svg>

tests/sample-documents/ISO-8859-1.sgml

@@ -0,0 +1,16 @@
+<!doctype linuxdoc system>
+<article>
+<title>B
+<author>A
+<date>2016-02-11
+<abstract> abstract </abstract>
+<toc>
+<sect>Introduction
+<p>
+<sect>Stuff.
+<p>
+Øh, Ýêåh, wë lóvèþ uß §ÓµË ISO-8859-1.
+<p>Pilcrow homage: ¶
+<sect>More-stuff.
+<p>
+</article>

tests/sample-documents/Linuxdoc-Larger/Linuxdoc-Larger.sgml

@@ -1,7 +1,7 @@
 <!doctype linuxdoc system>
 <article>
-<title>B
-<author>A
+<title>Linuxdoc Larger Document
+<author>Another Author
 <date>2016-02-11
 <abstract> abstract </abstract>
 <toc>

tests/sample-documents/Linuxdoc-Larger/images/another.png

@@ -0,0 +1 @@
+../../DocBookSGML-Larger/images/bullet.png

tests/sample-documents/Linuxdoc-Larger/images/tiny.png

@@ -0,0 +1 @@
+../../DocBookSGML-Larger/images/bullet.png

tests/sample-documents/docbook4xml-broken.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" standalone="no"?>
+<!DOCTYPE article PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
+                         "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
+<article>
+  <articleinfo>
+    <title>T</title>
+    <author><firstname>A</firstname><surname>B</surname></author>
+    <authorinitials>AB</authorinitials>
+    <revhistory> <revision>
+         <revnumber>v0.0</revnumber>
+         <date>2016-02-11</date>
+         <authorinitials>AB</authorinitials>
+          <revremark> Initial release.  </revremark>
+      </revision> </revhistory>
+    <abstract> <para> abstract </para> </abstract>
+  </articleinfo>
+
+  <sect1 id="intro">
+    <title>Intro</title>
+    <para>Text</para>
+    <sect2>
+      <title>Intro</title>
+      <para>Text</para>
+    </sect2>
+  </sect1>
+
+<!-- HERE IS THE PROBLEM!
+     You've got a ventricle in your article!
+     You should see a doctor about that.
+  -->
+</ventricle>

tests/sample-documents/markdown-simple.md

@@ -1 +0,0 @@
-No content.

tests/sample-documents/restructuredtext-simple.rst

@@ -1 +0,0 @@
-No content.

tests/test_cascadingconfig.py

@@ -1,5 +1,9 @@
+# -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
 
 import unittest
 import argparse
@@ -79,14 +83,13 @@ class CascadingConfigBasicTest(CCTestTools):
         ap.add_argument('--size', default=9, type=int)
 
         c = Namespace(
-          tag='tag',
-          argparser=ap,
-          argv=''.split(),
-          env=dict(),
-          cfg='',
-          exp_config=Namespace(size=9),
-          exp_args=[],
-        )
+            tag='tag',
+            argparser=ap,
+            argv=''.split(),
+            env=dict(),
+            cfg='',
+            exp_config=Namespace(size=9),
+            exp_args=[],)
 
         cc = CascadingConfig(c.tag, c.argparser, argv=c.argv, env=c.env)
         config, args = cc.parse()
@@ -98,14 +101,13 @@ class CascadingConfigBasicTest(CCTestTools):
         ap.add_argument('--size', default=9, type=int)
 
         c = Namespace(
-          tag='tag',
-          argparser=ap,
-          argv=''.split(),
-          env=dict(),
-          cfg='[tag]\nsize = 8',
-          exp_config=Namespace(size=8),
-          exp_args=[],
-        )
+            tag='tag',
+            argparser=ap,
+            argv=''.split(),
+            env=dict(),
+            cfg='[tag]\nsize = 8',
+            exp_config=Namespace(size=8),
+            exp_args=[],)
 
         self.writeconfig(c)
         c.env.setdefault('TAG_CONFIGFILE', c.configfile)
@@ -121,14 +123,13 @@ class CascadingConfigBasicTest(CCTestTools):
         import logging
         logging.getLogger().setLevel(logging.DEBUG)
         c = Namespace(
-          tag='tag',
-          argparser=ap,
-          argv=''.split(),
-          env=dict(),
-          cfg='[tag]\nsize = 8',
-          exp_config=Namespace(size=8),
-          exp_args=[],
-        )
+            tag='tag',
+            argparser=ap,
+            argv=''.split(),
+            env=dict(),
+            cfg='[tag]\nsize = 8',
+            exp_config=Namespace(size=8),
+            exp_args=[],)
         self.writeconfig(c)
         c.argv.extend(['--configfile', c.configfile])
         cc = CascadingConfig(c.tag, c.argparser, argv=c.argv, env=c.env)
@@ -143,14 +144,13 @@ class CascadingConfigBasicTest(CCTestTools):
         import logging
         logging.getLogger().setLevel(logging.DEBUG)
         c = Namespace(
-          tag='tag',
-          argparser=ap,
-          argv=''.split(),
-          env=dict(TAG_SIZE=7, ),
-          cfg='[tag]\nsize = 8',
-          exp_config=Namespace(size=7),
-          exp_args=[],
-        )
+            tag='tag',
+            argparser=ap,
+            argv=''.split(),
+            env=dict(TAG_SIZE=7, ),
+            cfg='[tag]\nsize = 8',
+            exp_config=Namespace(size=7),
+            exp_args=[],)
         self.writeconfig(c)
         c.argv.extend(['--configfile', c.configfile])
         cc = CascadingConfig(c.tag, c.argparser, argv=c.argv, env=c.env)
@@ -165,14 +165,13 @@ class CascadingConfigBasicTest(CCTestTools):
         import logging
         logging.getLogger().setLevel(logging.DEBUG)
         c = Namespace(
-          tag='tag',
-          argparser=ap,
-          argv='--size 6'.split(),
-          env=dict(TAG_SIZE=7, ),
-          cfg='[tag]\nsize = 8',
-          exp_config=Namespace(size=6),
-          exp_args=[],
-        )
+            tag='tag',
+            argparser=ap,
+            argv='--size 6'.split(),
+            env=dict(TAG_SIZE=7, ),
+            cfg='[tag]\nsize = 8',
+            exp_config=Namespace(size=6),
+            exp_args=[],)
         self.writeconfig(c)
         c.argv.extend(['--configfile', c.configfile])
         cc = CascadingConfig(c.tag, c.argparser, argv=c.argv, env=c.env)
@@ -184,14 +183,13 @@ class CascadingConfigBasicTest(CCTestTools):
         ap.add_argument('--source', default='', action='append', type=str)
 
         c = Namespace(
-          tag='tag',
-          argparser=ap,
-          argv=''.split(),
-          env=dict(),
-          cfg='',
-          exp_config=Namespace(source=''),
-          exp_args=[],
-        )
+            tag='tag',
+            argparser=ap,
+            argv=''.split(),
+            env=dict(),
+            cfg='',
+            exp_config=Namespace(source=''),
+            exp_args=[],)
         cc = CascadingConfig(c.tag, c.argparser, argv=c.argv, env=c.env)
         config, args = cc.parse()
         self.assertEqual(c.exp_config, config)
@@ -202,14 +200,13 @@ class CascadingConfigBasicTest(CCTestTools):
         ap.add_argument('--source', default='', action='append', type=str)
 
         c = Namespace(
-          tag='tag',
-          argparser=ap,
-          argv='--source /some/path'.split(),
-          env=dict(),
-          cfg='',
-          exp_config=Namespace(source=['/some/path']),
-          exp_args=[],
-        )
+            tag='tag',
+            argparser=ap,
+            argv='--source /some/path'.split(),
+            env=dict(),
+            cfg='',
+            exp_config=Namespace(source=['/some/path']),
+            exp_args=[],)
         cc = CascadingConfig(c.tag, c.argparser, argv=c.argv, env=c.env)
         config, args = cc.parse()
         self.assertEqual(c.exp_config, config)

tests/test_config.py

@@ -1,5 +1,10 @@
+# -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
+
 
 import unittest
 from argparse import Namespace
@@ -19,5 +24,12 @@ class TestConfigWorks(unittest.TestCase):
         config, args = collectconfiguration('tag', ['--pubdir', '.'])
         self.assertEqual(config.pubdir, '.')
 
+    def test_nonexistent_directory(self):
+        argv = ['--pubdir', '/path/to/nonexistent/directory']
+        with self.assertRaises(ValueError) as ecm:
+            config, args = collectconfiguration('tag', argv)
+        e = ecm.exception
+        self.assertTrue("/path/to/nonexistent/directory" in e.args[0])
+
 #
 # -- end of file

tests/test_driver.py

@@ -1,5 +1,10 @@
+# -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
+
 
 import io
 import os
@@ -7,6 +12,7 @@ import uuid
 import errno
 import codecs
 import random
+import unittest
 from tempfile import NamedTemporaryFile as ntf
 from argparse import Namespace
 
@@ -15,6 +21,7 @@ from tldp.typeguesser import knowndoctypes
 from tldp.inventory import stypes, status_types
 from tldp.sources import SourceDocument
 from tldp.outputs import OutputDirectory
+from tldp import VERSION
 
 # -- Test Data
 import example
@@ -23,7 +30,7 @@ import example
 import tldp.config
 import tldp.driver
 
-# -- variables
+# -- shorthand
 opj = os.path.join
 opd = os.path.dirname
 opa = os.path.abspath
@@ -44,7 +51,7 @@ class TestDriverDetail(TestInventoryBase):
         stdout = io.StringIO()
         tldp.driver.detail(c, docs, file=stdout)
         stdout.seek(0)
-        self.assertTrue('newer source' in stdout.read())
+        self.assertTrue('changed source' in stdout.read())
 
     def test_broken_detail_verbosity(self):
         c = self.config
@@ -122,6 +129,22 @@ class TestDriverShowStatustypes(TestToolsFilesystem):
         self.assertEqual(exitcode, os.EX_OK)
 
 
+class TestDriverShowVersion(unittest.TestCase):
+
+    def test_show_version(self):
+        stdout = io.StringIO()
+        result = tldp.driver.show_version(Namespace(), file=stdout)
+        self.assertEqual(result, os.EX_OK)
+        stdout.seek(0)
+        data = stdout.read().strip()
+        for status in status_types:
+            self.assertEqual(VERSION, data)
+
+    def test_run_statustypes(self):
+        exitcode = tldp.driver.run(['--version'])
+        self.assertEqual(exitcode, os.EX_OK)
+
+
 class TestDriverSummary(TestInventoryBase):
 
     def test_run_summary(self):
@@ -281,7 +304,7 @@ class Test_post_publish_cleanup(TestInventoryBase):
         tldp.driver.prepare_docs_build_mode(c, [doc])
         with open(opj(doc.dtworkingdir, 'annoyance-file.txt'), 'w'):
             pass
-        tldp.driver.post_publish_cleanup([doc])
+        tldp.driver.post_publish_cleanup([doc.dtworkingdir])
         self.assertTrue(os.path.isdir(doc.dtworkingdir))
 
 
@@ -355,19 +378,6 @@ class TestDriverRun(TestInventoryBase):
         exitcode = tldp.driver.run(argv)
         self.assertTrue('to --build' in exitcode)
 
-    def test_run_status_selection(self):
-        self.add_docbook4xml_xsl_to_config()
-        c = self.config
-        c.script = True
-        self.add_stale('Asciidoc-Stale-HOWTO', example.ex_asciidoc)
-        self.add_new('DocBook4XML-New-HOWTO', example.ex_docbook4xml)
-        argv = self.argv
-        argv.extend(['--publish', 'stale'])
-        exitcode = tldp.driver.run(argv)
-        self.assertEqual(exitcode, os.EX_OK)
-        inv = tldp.inventory.Inventory(c.pubdir, c.sourcedir)
-        self.assertEqual(1, len(inv.published.keys()))
-
 
 class TestDriverProcessSkips(TestInventoryBase):
 

tests/test_inventory.py

@@ -1,5 +1,10 @@
+# -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
+
 
 import random
 
@@ -41,7 +46,7 @@ class TestInventoryUsage(TestInventoryBase):
     def test_detect_status_published(self):
         c = self.config
         ex = random.choice(example.sources)
-        self.add_published('Frobnitz-HOWTO', ex)
+        self.add_published('Frobnitz-Published-HOWTO', ex)
         i = Inventory(c.pubdir, c.sourcedir)
         self.assertEqual(0, len(i.stale))
         self.assertEqual(1, len(i.published))
@@ -52,7 +57,7 @@ class TestInventoryUsage(TestInventoryBase):
     def test_detect_status_new(self):
         c = self.config
         ex = random.choice(example.sources)
-        self.add_new('Frobnitz-HOWTO', ex)
+        self.add_new('Frobnitz-New-HOWTO', ex)
         i = Inventory(c.pubdir, c.sourcedir)
         self.assertEqual(0, len(i.stale))
         self.assertEqual(0, len(i.published))
@@ -63,7 +68,7 @@ class TestInventoryUsage(TestInventoryBase):
     def test_detect_status_orphan(self):
         c = self.config
         ex = random.choice(example.sources)
-        self.add_orphan('Frobnitz-HOWTO', ex)
+        self.add_orphan('Frobnitz-Orphan-HOWTO', ex)
         i = Inventory(c.pubdir, c.sourcedir)
         self.assertEqual(0, len(i.stale))
         self.assertEqual(0, len(i.published))
@@ -74,7 +79,7 @@ class TestInventoryUsage(TestInventoryBase):
     def test_detect_status_stale(self):
         c = self.config
         ex = random.choice(example.sources)
-        self.add_stale('Frobnitz-HOWTO', ex)
+        self.add_stale('Frobnitz-Stale-HOWTO', ex)
         i = Inventory(c.pubdir, c.sourcedir)
         self.assertEqual(1, len(i.stale))
         self.assertEqual(1, len(i.published))
@@ -85,7 +90,7 @@ class TestInventoryUsage(TestInventoryBase):
     def test_detect_status_broken(self):
         c = self.config
         ex = random.choice(example.sources)
-        self.add_broken('Frobnitz-HOWTO', ex)
+        self.add_broken('Frobnitz-Broken-HOWTO', ex)
         i = Inventory(c.pubdir, c.sourcedir)
         self.assertEqual(0, len(i.stale))
         self.assertEqual(1, len(i.published))

tests/test_outputs.py

@@ -1,5 +1,9 @@
+# -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
 
 import os
 import errno
@@ -37,7 +41,7 @@ class TestOutputCollection(TestToolsFilesystem):
 
     def test_file_in_output_collection(self):
         reldir, absdir = self.adddir('collection')
-        self.addfile('collection', __file__,  stem='non-directory')
+        self.addfile('collection', __file__, stem='non-directory')
         oc = OutputCollection(absdir)
         self.assertEqual(0, len(oc))
 

tests/test_sources.py

@@ -1,5 +1,9 @@
+# -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
 
 import os
 import errno
@@ -143,11 +147,11 @@ class TestSourceDocument(TestToolsFilesystem):
     def test_init(self):
         for ex in example.sources:
             fullpath = ex.filename
-            fn = os.path.basename(fullpath)
+            fn = os.path.relpath(fullpath, start=example.sampledocs)
             doc = SourceDocument(fullpath)
             self.assertIsInstance(doc, SourceDocument)
             self.assertTrue(fn in str(doc))
-            self.assertTrue(fn in doc.statinfo)
+            self.assertTrue(fn in doc.md5sums)
 
     def test_fromfifo_should_fail(self):
         fifo = os.path.join(self.tempdir, 'fifofile')

tests/test_typeguesser.py

@@ -1,5 +1,9 @@
+# -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
 
 import os
 import codecs
@@ -13,6 +17,13 @@ import example
 from tldp.typeguesser import guess
 from tldp.doctypes.common import SignatureChecker
 
+# -- shorthand
+opj = os.path.join
+opd = os.path.dirname
+opa = os.path.abspath
+
+sampledocs = opj(opd(__file__), 'sample-documents')
+
 
 def genericGuessTest(content, ext):
         tf = ntf(prefix='tldp-guesser-test-', suffix=ext, delete=False)
@@ -26,6 +37,10 @@ def genericGuessTest(content, ext):
 
 class TestDoctypes(unittest.TestCase):
 
+    def testISO_8859_1(self):
+        dt = guess(opj(sampledocs, 'ISO-8859-1.sgml'))
+        self.assertIsNotNone(dt)
+
     def testDetectionBySignature(self):
         for ex in example.sources:
             if isinstance(ex.doctype, SignatureChecker):

tests/test_utils.py

@@ -1,5 +1,9 @@
+# -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
 
 import os
 import stat
@@ -18,6 +22,7 @@ from tldp.utils import statfile, statfiles, stem_and_ext
 from tldp.utils import arg_isexecutable, isexecutable
 from tldp.utils import arg_isreadablefile, isreadablefile
 from tldp.utils import arg_isdirectory, arg_isloglevel
+from tldp.utils import arg_isstr
 from tldp.utils import swapdirs
 
 
@@ -43,14 +48,31 @@ class Test_isreadablefile_and_friends(unittest.TestCase):
     def test_isreadablefile(self):
         f = ntf(prefix='readable-file')
         self.assertTrue(isreadablefile(f.name))
+        mode = os.stat(f.name).st_mode
         os.chmod(f.name, 0)
-        self.assertFalse(isreadablefile(f.name))
+        if 0 == os.getuid():
+            self.assertTrue(isreadablefile(f.name))
+        else:
+            self.assertFalse(isreadablefile(f.name))
+        os.chmod(f.name, mode)
 
     def test_arg_isreadablefile(self):
         f = ntf(prefix='readable-file')
         self.assertEqual(f.name, arg_isreadablefile(f.name))
+        mode = os.stat(f.name).st_mode
         os.chmod(f.name, 0)
-        self.assertIsNone(arg_isreadablefile(f.name))
+        if 0 == os.getuid():
+            self.assertEqual(f.name, arg_isreadablefile(f.name))
+        else:
+            self.assertIsNone(arg_isreadablefile(f.name))
+        os.chmod(f.name, mode)
+
+
+class Test_arg_isstr(unittest.TestCase):
+
+    def test_arg_isstr(self):
+        self.assertEqual('s', arg_isstr('s'))
+        self.assertEqual(None, arg_isstr(7))
 
 
 class Test_arg_isloglevel(unittest.TestCase):
@@ -146,7 +168,8 @@ class Test_statfiles(unittest.TestCase):
         here = os.path.dirname(os.path.abspath(__file__))
         statinfo = statfiles(here, relative=here)
         self.assertIsInstance(statinfo, dict)
-        self.assertTrue(os.path.basename('sample-documents') in statinfo)
+        adoc = 'sample-documents/asciidoc-complete.txt'
+        self.assertTrue(adoc in statinfo)
 
     def test_statfiles_dir_rel(self):
         here = os.path.dirname(os.path.abspath(__file__))
@@ -193,9 +216,12 @@ class Test_statfile(TestToolsFilesystem):
         f = ntf(dir=self.tempdir)
         omode = os.stat(self.tempdir).st_mode
         os.chmod(self.tempdir, 0)
-        with self.assertRaises(OSError):
-            statfile(f.name)
-        os.chmod(self.tempdir, omode)
+        if 0 != os.getuid():
+            with self.assertRaises(Exception) as ecm:
+                statfile(f.name)
+            e = ecm.exception
+            self.assertIn(e.errno, (errno.EPERM, errno.EACCES))
+            os.chmod(self.tempdir, omode)
         stbuf = statfile(f.name)
         self.assertIsInstance(stbuf, posix.stat_result)
 
@@ -245,11 +271,5 @@ class Test_swapdirs(TestToolsFilesystem):
         self.assertTrue(os.path.exists(b))
         self.assertTrue(os.path.exists(bfile))
 
-
-class Test_att_statinfo(unittest.TestCase):
-
-    def test_max_mtime(self):
-        pass
-
 #
 # -- end of file

tests/tldptesttools.py

@@ -1,8 +1,11 @@
+# -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
 
 import os
-import time
 import codecs
 import random
 import shutil
@@ -12,6 +15,7 @@ from tempfile import NamedTemporaryFile as ntf
 
 import tldp.config
 from tldp.outputs import OutputNamingConvention
+from tldp.utils import writemd5sums, md5file
 
 # -- short names
 #
@@ -100,7 +104,7 @@ class CCTestTools(unittest.TestCase):
         tf = ntf(prefix=case.tag, suffix='.cfg', dir=self.tempdir, delete=False)
         tf.close()
         with codecs.open(tf.name, 'w', encoding='utf-8') as f:
-          f.write(case.cfg)
+            f.write(case.cfg)
         case.configfile = tf.name
 
 
@@ -110,18 +114,14 @@ class TestOutputDirSkeleton(OutputNamingConvention):
         if not os.path.isdir(self.dirname):
             os.mkdir(self.dirname)
 
-    def create_expected_docs(self, func=None):
+    def create_md5sum_file(self, md5s):
+        writemd5sums(self.MD5SUMS, md5s)
+
+    def create_expected_docs(self):
         for name in self.expected:
             fname = getattr(self, name)
             with open(fname, 'w'):
                 pass
-            if func:
-                func(fname)
-
-    def create_stale_expected_docs(self):
-        def thirtysecondsago(fname):
-            os.utime(fname, (time.time() - 30, time.time() - 30))
-        self.create_expected_docs(func=thirtysecondsago)
 
 
 class TestSourceDocSkeleton(object):
@@ -134,6 +134,22 @@ class TestSourceDocSkeleton(object):
         self.dirname = dirname
         if not os.path.isdir(self.dirname):
             os.mkdir(self.dirname)
+        self.md5s = dict()
+
+    def copytree(self, source):
+        dst = opj(self.dirname, opb(source))
+        shutil.copytree(source, dst)
+
+    def create_stale(self, fname):
+        l = list(self.md5s[fname])
+        random.shuffle(l)
+        if l == self.md5s[fname]:
+            self.invalidate_checksum(fname)
+        self.md5s[fname] = ''.join(l)
+
+    @property
+    def md5sums(self):
+        return self.md5s
 
     def addsourcefile(self, filename, content):
         fname = os.path.join(self.dirname, filename)
@@ -142,6 +158,8 @@ class TestSourceDocSkeleton(object):
         else:
             with codecs.open(fname, 'w', encoding='utf-8') as f:
                 f.write(content)
+        relpath = os.path.relpath(fname, start=self.dirname)
+        self.md5s[relpath] = md5file(fname)
 
 
 class TestInventoryBase(unittest.TestCase):
@@ -176,19 +194,24 @@ class TestInventoryBase(unittest.TestCase):
 
     def add_stale(self, stem, ex):
         c = self.config
+        mysource = TestSourceDocSkeleton(c.sourcedir)
+        fname = stem + ex.ext
+        mysource.addsourcefile(fname, ex.filename)
+        mysource.create_stale(fname)
         myoutput = TestOutputDirSkeleton(os.path.join(c.pubdir, stem), stem)
         myoutput.mkdir()
-        myoutput.create_stale_expected_docs()
-        mysource = TestSourceDocSkeleton(c.sourcedir)
-        mysource.addsourcefile(stem + ex.ext, ex.filename)
+        myoutput.create_expected_docs()
+        myoutput.create_md5sum_file(mysource.md5sums)
 
     def add_broken(self, stem, ex):
         c = self.config
         mysource = TestSourceDocSkeleton(c.sourcedir)
-        mysource.addsourcefile(stem + ex.ext, ex.filename)
+        fname = stem + ex.ext
+        mysource.addsourcefile(fname, ex.filename)
         myoutput = TestOutputDirSkeleton(os.path.join(c.pubdir, stem), stem)
         myoutput.mkdir()
         myoutput.create_expected_docs()
+        myoutput.create_md5sum_file(mysource.md5sums)
         prop = random.choice(myoutput.expected)
         fname = getattr(myoutput, prop, None)
         assert fname is not None
@@ -223,6 +246,7 @@ class TestInventoryBase(unittest.TestCase):
         myoutput = TestOutputDirSkeleton(os.path.join(c.pubdir, stem), stem)
         myoutput.mkdir()
         myoutput.create_expected_docs()
+        myoutput.create_md5sum_file(mysource.md5sums)
 
     def add_docbooksgml_support_to_config(self):
         c = self.config

tldp/__init__.py

@@ -1,8 +1,13 @@
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import
+from __future__ import unicode_literals
 
 import tldp.config
 import tldp.outputs
 import tldp.sources
 import tldp.inventory
+
+VERSION = "0.7.15"

tldp/cascadingconfig.py

@@ -1,5 +1,7 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals

tldp/config.py

@@ -1,12 +1,17 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
 
+import os
+import argparse
+
 import logging
 
-from tldp.utils import arg_isdirectory, arg_isloglevel, arg_isreadablefile
+from tldp.utils import arg_isloglevel, arg_isreadablefile
 from tldp.cascadingconfig import CascadingConfig, DefaultFreeArgumentParser
 
 import tldp.typeguesser
@@ -16,69 +21,136 @@ logger = logging.getLogger(__name__)
 DEFAULT_CONFIGFILE = '/etc/ldptool/ldptool.ini'
 
 
+class DirectoriesExist(argparse._AppendAction):
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        if not os.path.isdir(values):
+            message = "No such directory: %r for option %r, aborting..."
+            message = message % (values, option_string)
+            logger.critical(message)
+            raise ValueError(message)
+        items = getattr(namespace, self.dest, [])
+        items.append(values)
+        setattr(namespace, self.dest, items)
+
+
+class DirectoryExists(argparse._StoreAction):
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        if not os.path.isdir(values):
+            message = "No such directory: %r for option %r, aborting..."
+            message = message % (values, option_string)
+            logger.critical(message)
+            raise ValueError(message)
+        setattr(namespace, self.dest, values)
+
+
+class StoreTrueOrNargBool(argparse._StoreAction):
+
+    _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
+                       '0': False, 'no': False, 'false': False, 'off': False}
+
+    def __init__(self, *args, **kwargs):
+        super(argparse._StoreAction, self).__init__(*args, **kwargs)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        if values is None:
+            setattr(namespace, self.dest, True)
+        else:
+            boolval = self._boolean_states.get(values.lower(), None)
+            if boolval is None:
+                message = "Non-boolean value: %r for option %r, aborting..."
+                message = message % (values, option_string)
+                logger.critical(message)
+                raise ValueError(message)
+            else:
+                setattr(namespace, self.dest, boolval)
+
+
 def collectconfiguration(tag, argv):
+    '''main specification of command-line (and config file) shape'''
+
     ap = DefaultFreeArgumentParser()
-
-    g = ap.add_mutually_exclusive_group()
-    g.add_argument('--build',
-                   '-b',
-                   action='store_true', default=False,
-                   help='build LDP documentation [%(default)s]')
-    g.add_argument('--publish',
-                   '-p',
-                   action='store_true', default=False,
-                   help='build and publish LDP documentation [%(default)s]')
-    g.add_argument('--script',
-                   '-S',
-                   action='store_true', default=False,
-                   help='dump runnable script [%(default)s]')
-    g.add_argument('--detail', '--list',
-                   '-l',
-                   action='store_true', default=False,
-                   help='list elements of LDP system [%(default)s]')
-    g.add_argument('--summary',
-                   '-t',
-                   action='store_true', default=False,
-                   help='dump inventory summary report [%(default)s]')
-    g.add_argument('--doctypes', '--formats', '--format',
-                   '--list-doctypes', '--list-formats',
-                   '-T',
-                   action='store_true', default=False,
-                   help='show supported doctypes [%(default)s]')
-    g.add_argument('--statustypes', '--list-statustypes',
-                   action='store_true', default=False,
-                   help='show status types and classes [%(default)s]')
-
-    ap.add_argument('--verbose',
-                    action='store_true', default=False,
-                    help='more info in --list/--detail [%(default)s]')
-    ap.add_argument('--loglevel',
-                    default=logging.ERROR, type=arg_isloglevel,
-                    help='set the loglevel')
-    ap.add_argument('--skip',
-                    default=[], action='append', type=str,
-                    help='skip this stem during processing')
-    ap.add_argument('--resources',
-                    default=['images', 'resources'], action='append', type=str,
-                    help='subdirs to copy during build [%(default)s]')
     ap.add_argument('--sourcedir', '--source-dir', '--source-directory',
                     '-s',
-                    action='append', default='', type=arg_isdirectory,
+                    default=[], action=DirectoriesExist,
                     help='a directory containing LDP source documents')
+
     ap.add_argument('--pubdir', '--output', '--outputdir', '--outdir',
                     '-o',
-                    default=None, type=arg_isdirectory,
+                    default=None, action=DirectoryExists,
                     help='a directory containing LDP output documents')
+
     ap.add_argument('--builddir', '--build-dir', '--build-directory',
                     '-d',
-                    default=None, type=arg_isdirectory,
+                    default=None, action=DirectoryExists,
                     help='a scratch directory used for building')
+
     ap.add_argument('--configfile', '--config-file', '--cfg',
                     '-c',
                     default=DEFAULT_CONFIGFILE,
                     type=arg_isreadablefile,
                     help='a configuration file')
 
+    ap.add_argument('--loglevel',
+                    default=logging.ERROR, type=arg_isloglevel,
+                    help='set the loglevel')
+
+    ap.add_argument('--verbose',
+                    action=StoreTrueOrNargBool, nargs='?', default=False,
+                    help='more info in --list/--detail [%(default)s]')
+
+    ap.add_argument('--skip',
+                    default=[], action='append', type=str,
+                    help='skip this stem during processing')
+
+    ap.add_argument('--resources',
+                    default=['images', 'resources'], action='append', type=str,
+                    help='subdirs to copy during build [%(default)s]')
+
+    # -- and the distinct, mutually exclusive actions this script can perform
+    #
+    g = ap.add_mutually_exclusive_group()
+    g.add_argument('--publish',
+                   '-p',
+                   action='store_true', default=False,
+                   help='build and publish LDP documentation [%(default)s]')
+
+    g.add_argument('--build',
+                   '-b',
+                   action='store_true', default=False,
+                   help='build LDP documentation [%(default)s]')
+
+    g.add_argument('--script',
+                   '-S',
+                   action='store_true', default=False,
+                   help='dump runnable script [%(default)s]')
+
+    g.add_argument('--detail', '--list',
+                   '-l',
+                   action='store_true', default=False,
+                   help='list elements of LDP system [%(default)s]')
+
+    g.add_argument('--summary',
+                   '-t',
+                   action='store_true', default=False,
+                   help='dump inventory summary report [%(default)s]')
+
+    g.add_argument('--doctypes', '--formats', '--format',
+                   '--list-doctypes', '--list-formats',
+                   '-T',
+                   action='store_true', default=False,
+                   help='show supported doctypes [%(default)s]')
+
+    g.add_argument('--statustypes', '--list-statustypes',
+                   action='store_true', default=False,
+                   help='show status types and classes [%(default)s]')
+
+    g.add_argument('--version',
+                   '-V',
+                   action='store_true', default=False,
+                   help='print out the version number [%(default)s]')
+
     # -- collect up the distributed configuration fragments
     #
     for cls in tldp.typeguesser.knowndoctypes:

tldp/doctypes/__init__.py

@@ -1,7 +1,11 @@
-# from .rst import RestructuredText
-# from .markdown import Markdown
-from .asciidoc import Asciidoc
-from .linuxdoc import Linuxdoc
-from .docbooksgml import DocbookSGML
-from .docbook4xml import Docbook4XML
-from .docbook5xml import Docbook5XML
+# -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
+
+from __future__ import absolute_import
+
+from tldp.doctypes.asciidoc import Asciidoc
+from tldp.doctypes.linuxdoc import Linuxdoc
+from tldp.doctypes.docbooksgml import DocbookSGML
+from tldp.doctypes.docbook4xml import Docbook4XML
+from tldp.doctypes.docbook5xml import Docbook5XML

tldp/doctypes/asciidoc.py

@@ -1,5 +1,7 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals

tldp/doctypes/common.py

@@ -1,5 +1,7 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
@@ -7,6 +9,7 @@ from __future__ import unicode_literals
 import os
 import sys
 import stat
+import time
 import errno
 import codecs
 import shutil
@@ -16,7 +19,7 @@ from tempfile import NamedTemporaryFile as ntf
 from functools import wraps
 import networkx as nx
 
-from tldp.utils import execute, logtimings
+from tldp.utils import execute, logtimings, writemd5sums
 
 logger = logging.getLogger(__name__)
 
@@ -140,11 +143,36 @@ class BaseDoctype(object):
             s = '''
 # - - - - - {source.stem} - - - - - -
 
-            cd -- "{output.dirname}"'''
+cd -- "{output.dirname}"'''
             return self.shellscript(s, **kwargs)
         os.chdir(self.output.dirname)
         return True
 
+    def generate_md5sums(self, **kwargs):
+        logger.debug("%s generating MD5SUMS in %s.",
+                     self.output.stem, self.output.dirname)
+        timestr = time.strftime('%F-%T', time.gmtime())
+        md5file = self.output.MD5SUMS
+        if self.config.script:
+            l = list()
+            for fname, hashval in sorted(self.source.md5sums.items()):
+                l.append('# {}  {}'.format(hashval, fname))
+            md5s = '\n'.join(l)
+            s = '''# -- MD5SUMS file from source tree at {}
+#
+# md5sum > {} -- {}
+#
+{}
+#'''
+            s = s.format(timestr,
+                         md5file,
+                         ' '.join(self.source.md5sums.keys()),
+                         md5s)
+            return self.shellscript(s, **kwargs)
+        header = '# -- MD5SUMS for {}'.format(self.source.stem)
+        writemd5sums(md5file, self.source.md5sums, header=header)
+        return True
+
     def copy_static_resources(self, **kwargs):
         logger.debug("%s copy resources %s.",
                      self.output.stem, self.output.dirname)
@@ -233,6 +261,7 @@ class BaseDoctype(object):
                  'clear_output',
                  'mkdir_output',
                  'chdir_output',
+                 'generate_md5sums',
                  'copy_static_resources',
                  ]
 

tldp/doctypes/docbook4xml.py

@@ -1,5 +1,7 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
@@ -32,10 +34,10 @@ def xslsingle_finder():
 
 def xslprint_finder():
     l = ['http://docbook.sourceforge.net/release/xsl/current/fo/docbook.xsl',
-         #'/usr/share/xml/docbook/stylesheet/ldp/fo/tldp-print.xsl',
+         # '/usr/share/xml/docbook/stylesheet/ldp/fo/tldp-print.xsl',
          ]
     return l[0]
-    #return firstfoundfile(l)
+    # return firstfoundfile(l)
 
 
 class Docbook4XML(BaseDoctype, SignatureChecker):

tldp/doctypes/docbook5xml.py

@@ -1,5 +1,7 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals

tldp/doctypes/docbooksgml.py

@@ -1,5 +1,7 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
@@ -127,7 +129,9 @@ class DocbookSGML(BaseDoctype, SignatureChecker):
                 logger.error("%s expected to find %s",
                              self.source.stem, self.output.dirname)
                 return False
-        s = '''find . -mindepth 1 -maxdepth 1 -not -type d -delete -print'''
+        preserve = os.path.basename(self.output.MD5SUMS)
+        s = '''find . -mindepth 1 -maxdepth 1 -not -type d -not -name {} -delete -print'''
+        s = s.format(preserve)
         return self.shellscript(s, **kwargs)
 
     @depends(cleaned_indexsgml)

tldp/doctypes/example.py

@@ -1,5 +1,7 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals

tldp/doctypes/linuxdoc.py

@@ -1,5 +1,7 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals

tldp/doctypes/markdown.py

@@ -1,5 +1,7 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals

tldp/doctypes/rst.py

@@ -1,5 +1,7 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals

tldp/driver.py

@@ -1,5 +1,7 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
@@ -22,6 +24,7 @@ from tldp.config import collectconfiguration
 from tldp.utils import arg_isloglevel, arg_isdirectory
 from tldp.utils import swapdirs, sameFilesystem
 from tldp.doctypes.common import preamble, postamble
+from tldp import VERSION
 
 # -- Don't freak out with IOError when our STDOUT, handled with
 #    head, sed, awk, grep, etc; and, also deal with a user's ctrl-C
@@ -31,7 +34,7 @@ signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 signal.signal(signal.SIGINT, signal.SIG_DFL)
 
 logformat = '%(levelname)-9s %(name)s %(filename)s#%(lineno)s ' \
-     + '%(funcName)s %(message)s'
+    + '%(funcName)s %(message)s'
 logging.basicConfig(stream=sys.stderr, format=logformat, level=logging.ERROR)
 logger = logging.getLogger(__name__)
 
@@ -50,6 +53,11 @@ ERR_UNKNOWNARGS = "Unknown arguments received: "
 ERR_EXTRAARGS = "Extra arguments received: "
 
 
+def show_version(config, *args, **kwargs):
+    file = kwargs.get('file', sys.stdout)
+    print(VERSION, file=file)
+    return os.EX_OK
+
 def show_doctypes(config, *args, **kwargs):
     if args:
         return ERR_EXTRAARGS + ' '.join(args)
@@ -106,10 +114,11 @@ def summary(config, *args, **kwargs):
     if inv is None:
         inv = Inventory(config.pubdir, config.sourcedir)
     width = Namespace()
-    width.doctype = max([len(x.formatname) for x in knowndoctypes])
+    width.doctype = max([len(x.__name__) for x in knowndoctypes])
     width.status = max([len(x) for x in status_types])
     width.count = len(str(len(inv.source.keys())))
-    print('By Status Type', '--------------', sep='\n', file=file)
+    print('By Document Status (STATUS)', '---------------------------',
+          sep='\n', file=file)
     for status in status_types:
         count = len(getattr(inv, status, 0))
         s = '{0:{w.status}}  {1:{w.count}}  '.format(status, count, w=width)
@@ -128,10 +137,11 @@ def summary(config, *args, **kwargs):
                 if abbrev:
                     s = s + ', and %d more ...' % (len(abbrev))
             print(s, file=file)
-    print('', 'By Document Type', '----------------', sep='\n', file=file)
+    print('', 'By Document Type (DOCTYPE)', '--------------------------',
+          sep='\n', file=file)
     summarybytype = collections.defaultdict(list)
     for doc in inv.source.values():
-        name = doc.doctype.formatname
+        name = doc.doctype.__name__
         summarybytype[name].append(doc.stem)
     for doctype, docs in summarybytype.items():
         count = len(docs)
@@ -158,7 +168,7 @@ def summary(config, *args, **kwargs):
 def detail(config, docs, **kwargs):
     file = kwargs.get('file', sys.stdout)
     width = Namespace()
-    width.doctype = max([len(x.formatname) for x in knowndoctypes])
+    width.doctype = max([len(x.__name__) for x in knowndoctypes])
     width.status = max([len(x) for x in status_types])
     width.stem = max([len(x.stem) for x in docs])
     # -- if user just said "list" with no args, then give the user something
@@ -228,13 +238,12 @@ def create_dtworkingdir(config, docs):
     return True, None
 
 
-def post_publish_cleanup(docs):
-    '''clean up any doctype directories left in --builddir'''
-    dtworkingdirs = set([x.dtworkingdir for x in docs])
-    for d in dtworkingdirs:
+def post_publish_cleanup(workingdirs):
+    '''clean up empty directories left under --builddir'''
+    for d in workingdirs:
         if os.path.isdir(d):
             try:
-                logger.debug("removing doctype build dir %s", d)
+                logger.debug("removing build dir %s", d)
                 os.rmdir(d)
             except OSError as e:
                 if e.errno != errno.ENOTEMPTY:
@@ -273,8 +282,10 @@ def docbuild(config, docs, **kwargs):
     for x, source in enumerate(docs, 1):
         working = source.working
         runner = source.doctype(source=source, output=working, config=config)
-        logger.info("%s (%d of %d) initiating build",
-                    source.stem, x, len(docs))
+        status = 'progress, %d failures, %d successes'
+        status = status % (result.count(False), result.count(True),)
+        logger.info("%s (%d of %d) initiating build [%s]",
+                    source.stem, x, len(docs), status)
         result.append(runner.generate(**kwargs))
     if all(result):
         buildsuccess = True
@@ -324,18 +335,21 @@ def build(config, docs, **kwargs):
 def publish(config, docs, **kwargs):
     config.build = True
     result = build(config, docs, **kwargs)
-    if result == os.EX_OK:
-        for x, source in enumerate(docs, 1):
-            logger.info("Publishing (%d of %d) to %s.",
-                        x, len(docs), source.output.dirname)
-            # -- swapdirs must raise an error if there are problems
-            #
-            swapdirs(source.working.dirname, source.output.dirname)
-            if os.path.isdir(source.working.dirname):
-                logger.debug("%s removing old directory %s",
-                             source.stem, source.working.dirname)
-                shutil.rmtree(source.working.dirname)
-        post_publish_cleanup(docs)
+    if result != os.EX_OK:
+        return result
+    for x, source in enumerate(docs, 1):
+        logger.info("Publishing (%d of %d) to %s.",
+                    x, len(docs), source.output.dirname)
+        # -- swapdirs must raise an error if there are problems
+        #
+        swapdirs(source.working.dirname, source.output.dirname)
+        if os.path.isdir(source.working.dirname):
+            logger.debug("%s removing old directory %s",
+                         source.stem, source.working.dirname)
+            shutil.rmtree(source.working.dirname)
+    workingdirs = list(set([x.dtworkingdir for x in docs]))
+    workingdirs.append(config.builddir)
+    post_publish_cleanup(workingdirs)
     return os.EX_OK
 
 
@@ -500,6 +514,9 @@ def collectWorkset(config, args):
 
 def handleArgs(config, args):
 
+    if config.version:
+        return show_version(config, *args)
+
     if config.doctypes:
         return show_doctypes(config, *args)
 

tldp/inventory.py

@@ -1,5 +1,7 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
@@ -8,8 +10,6 @@ import copy
 import logging
 from collections import OrderedDict
 
-from tldp.utils import max_mtime, mtime_gt
-
 from tldp.sources import SourceCollection
 from tldp.outputs import OutputCollection
 
@@ -73,8 +73,7 @@ class Inventory(object):
                len(self.orphan),
                len(self.new),
                len(self.stale),
-               len(self.broken),
-               )
+               len(self.broken),)
 
     def __init__(self, pubdir, sourcedirs):
         '''construct an Inventory
@@ -123,23 +122,7 @@ class Inventory(object):
         self.published = s
         logger.debug("Identified %d published documents.", len(self.published))
 
-        # -- stale identification
-        #
-        self.stale = SourceCollection()
-        for stem, sdoc in s.items():
-            odoc = sdoc.output
-            mtime = max_mtime(odoc.statinfo)
-            fset = mtime_gt(mtime, sdoc.statinfo)
-            if fset:
-                sdoc.newer = fset
-                for f in fset:
-                    logger.debug("%s found updated source file %s", stem, f)
-                odoc.status = sdoc.status = 'stale'
-                self.stale[stem] = sdoc
-        logger.debug("Identified %d stale documents: %r.", len(self.stale),
-                     self.stale.keys())
-
-        # -- stale identification
+        # -- broken identification
         #
         self.broken = SourceCollection()
         for stem, sdoc in s.items():
@@ -149,6 +132,31 @@ class Inventory(object):
         logger.debug("Identified %d broken documents: %r.", len(self.broken),
                      self.broken.keys())
 
+        # -- stale identification
+        #
+        self.stale = SourceCollection()
+        for stem, sdoc in s.items():
+            odoc = sdoc.output
+            omd5, smd5 = odoc.md5sums, sdoc.md5sums
+            if omd5 != smd5:
+                logger.debug("%s differing MD5 sets %r %r", stem, smd5, omd5)
+                changed = set()
+                for gone in set(omd5.keys()).difference(smd5.keys()):
+                    logger.debug("%s gone %s", stem, gone)
+                    changed.add(('gone', gone))
+                for new in set(smd5.keys()).difference(omd5.keys()):
+                    changed.add(('new', new))
+                for sfn in set(smd5.keys()).intersection(omd5.keys()):
+                    if smd5[sfn] != omd5[sfn]:
+                        changed.add(('changed', sfn))
+                for why, sfn in changed:
+                    logger.debug("%s differing source %s (%s)", stem, sfn, why)
+                odoc.status = sdoc.status = 'stale'
+                sdoc.differing = changed
+                self.stale[stem] = sdoc
+        logger.debug("Identified %d stale documents: %r.", len(self.stale),
+                     self.stale.keys())
+
     def getByStatusClass(self, status_class):
         desired = status_classes.get(status_class, None)
         assert isinstance(desired, list)

tldp/ldpcollection.py

@@ -1,12 +1,19 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 
-import collections
+import sys
+
+if sys.version_info[:2] >= (3, 8):  # pragma: no cover
+    from collections.abc import MutableMapping
+else:  # pragma: no cover
+    from collections import MutableMapping
 
 
-class LDPDocumentCollection(collections.MutableMapping):
+class LDPDocumentCollection(MutableMapping):
     '''a dict-like container for DocumentCollection objects
 
     Intended to be subclassed.

tldp/outputs.py

@@ -1,5 +1,7 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
@@ -7,10 +9,11 @@ from __future__ import unicode_literals
 import os
 import sys
 import errno
+import codecs
 import logging
 
 from tldp.ldpcollection import LDPDocumentCollection
-from tldp.utils import logdir, statfiles
+from tldp.utils import logdir
 
 logger = logging.getLogger(__name__)
 
@@ -31,6 +34,10 @@ class OutputNamingConvention(object):
         self.dirname = dirname
         self.stem = stem
 
+    @property
+    def MD5SUMS(self):
+        return os.path.join(self.dirname, '.LDP-source-MD5SUMS')
+
     @property
     def name_txt(self):
         return os.path.join(self.dirname, self.stem + '.txt')
@@ -84,6 +91,21 @@ class OutputNamingConvention(object):
                 missing.add(name)
         return missing
 
+    @property
+    def md5sums(self):
+        d = dict()
+        try:
+            with codecs.open(self.MD5SUMS, encoding='utf-8') as f:
+                for line in f:
+                    if line.startswith('#'):
+                        continue
+                    hashval, fname = line.strip().split()
+                    d[fname] = hashval
+        except IOError as e:
+            if e.errno != errno.ENOENT:
+                raise
+        return d
+
 
 class OutputDirectory(OutputNamingConvention):
     '''A class providing a container for each set of output documents
@@ -120,7 +142,6 @@ class OutputDirectory(OutputNamingConvention):
         if not os.path.isdir(parent):
             logger.critical("Missing output collection directory %s.", parent)
             raise IOError(errno.ENOENT, os.strerror(errno.ENOENT), parent)
-        self.statinfo = statfiles(self.dirname, relative=self.dirname)
         self.status = 'output'
         self.source = source
         self.logdir = os.path.join(self.dirname, logdir)

tldp/sources.py

@@ -1,5 +1,7 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
@@ -11,11 +13,13 @@ import logging
 
 from tldp.ldpcollection import LDPDocumentCollection
 
-from tldp.utils import statfiles, stem_and_ext
+from tldp.utils import md5files, stem_and_ext
 from tldp.typeguesser import guess, knownextensions
 
 logger = logging.getLogger(__name__)
 
+IGNORABLE_SOURCE = ('index.sgml')
+
 
 def scansourcedirs(dirnames):
     '''return a dict() of all SourceDocuments discovered in dirnames
@@ -82,6 +86,8 @@ def scansourcedirs(dirnames):
 def arg_issourcedoc(filename):
     filename = os.path.abspath(filename)
     if os.path.isfile(filename):
+        if os.path.basename(filename) in IGNORABLE_SOURCE:
+            return None
         return filename
     elif os.path.isdir(filename):
         return sourcedoc_fromdir(filename)
@@ -186,20 +192,21 @@ class SourceDocument(object):
         self.status = 'source'
         self.output = None
         self.working = None
-        self.newer = set()
+        self.differing = set()
         self.dirname, self.basename = os.path.split(self.filename)
         self.stem, self.ext = stem_and_ext(self.basename)
         parentbase = os.path.basename(self.dirname)
         logger.debug("%s found source %s", self.stem, self.filename)
         if parentbase == self.stem:
-            self.statinfo = statfiles(self.dirname, relative=self.dirname)
+            parentdir = os.path.dirname(self.dirname)
+            self.md5sums = md5files(self.dirname, relative=parentdir)
         else:
-            self.statinfo = statfiles(self.filename, relative=self.dirname)
+            self.md5sums = md5files(self.filename, relative=self.dirname)
 
     def detail(self, widths, verbose, file=sys.stdout):
         '''produce a small tabular output about the document'''
         template = ' '.join(('{s.status:{w.status}}',
-                             '{s.doctype.formatname:{w.doctype}}',
+                             '{s.doctype.__name__:{w.doctype}}',
                              '{s.stem:{w.stem}}'))
         outstr = template.format(s=self, w=widths)
         print(outstr, file=file)
@@ -209,9 +216,9 @@ class SourceDocument(object):
                 print('      output dir {}'.format(self.output.dirname),
                       file=file)
             print('     source file {}'.format(self.filename), file=file)
-            for f in sorted(self.newer):
+            for why, f in sorted(self.differing):
                 fname = os.path.join(self.dirname, f)
-                print('    newer source {}'.format(fname), file=file)
+                print('  {:>7} source {}'.format(why, fname), file=file)
             if self.output:
                 for f in sorted(self.output.missing):
                     print('  missing output {}'.format(f), file=file)

tldp/typeguesser.py

@@ -1,10 +1,11 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 
 import os
-import errno
 import codecs
 import inspect
 import logging
@@ -56,7 +57,7 @@ def guess(fname):
     '''
     try:
         stem, ext = os.path.splitext(fname)
-    except AttributeError:
+    except (AttributeError, TypeError):
         return None
 
     if not ext:

tldp/utils.py

@@ -1,15 +1,16 @@
 #! /usr/bin/python
 # -*- coding: utf8 -*-
+#
+# Copyright (c) 2016 Linux Documentation Project
 
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
 
 import os
-import io
 import time
 import errno
 import codecs
-import operator
+import hashlib
 import subprocess
 import functools
 from functools import wraps
@@ -47,7 +48,7 @@ def firstfoundfile(locations):
     return None
 
 
-def arg_isloglevel(l):
+def arg_isloglevel(l, defaultlevel=logging.ERROR):
     try:
         level = int(l)
         return level
@@ -55,7 +56,7 @@ def arg_isloglevel(l):
         pass
     level = getattr(logging, l.upper(), None)
     if not level:
-        level = logging.ERROR
+        level = defaultlevel
     return level
 
 
@@ -229,7 +230,7 @@ def isstr(s):
         unicode
         stringy = (str, unicode)
     except NameError:
-        stringy = (str,)
+        stringy = (str,)  # -- python3
     return isinstance(s, stringy)
 
 
@@ -249,6 +250,27 @@ http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python/37
     return None
 
 
+def writemd5sums(fname, md5s, header=None):
+    '''write an MD5SUM file from [(filename, MD5), ...]'''
+    with codecs.open(fname, 'w', encoding='utf-8') as file:
+        if header:
+            print(header, file=file)
+        for fname, hashval in sorted(md5s.items()):
+            print(hashval + '  ' + fname, file=file)
+
+
+def md5file(name):
+    '''return MD5 hash for a single file name'''
+    with open(name, 'rb') as f:
+        bs = f.read()
+    md5 = hashlib.md5(bs).hexdigest()
+    try:
+        md5 = unicode(md5)
+    except NameError:
+        pass  # -- python3
+    return md5
+
+
 def statfile(name):
     '''return posix.stat_result (or None) for a single file name'''
     try:
@@ -260,7 +282,24 @@ def statfile(name):
     return st
 
 
+def md5files(name, relative=None):
+    '''get all of the MD5s for files from here downtree'''
+    return fileinfo(name, relative=relative, func=md5file)
+
+
 def statfiles(name, relative=None):
+    '''
+    >>> statfiles('./docs/x509').keys()
+    ['./docs/x509/tutorial.rst', './docs/x509/reference.rst', './docs/x509/index.rst']
+    >>> statfiles('./docs/x509', relative='./').keys()
+    ['docs/x509/reference.rst', 'docs/x509/tutorial.rst', 'docs/x509/index.rst']
+    >>> statfiles('./docs/x509', relative='./docs/x509/').keys()
+    ['index.rst', 'tutorial.rst', 'reference.rst']
+    '''
+    return fileinfo(name, relative=relative, func=statfile)
+
+
+def fileinfo(name, relative=None, func=statfile):
     '''return a dict() with keys being filenames and posix.stat_result values
 
     Required:
@@ -283,27 +322,18 @@ def statfiles(name, relative=None):
       least we can try to rely on them as best we can--mostly, by just
       excluding any files (in the output dict()) which did not return a valid
       posix.stat_result.
-
-    Examples:
-
-    >>> statfiles('./docs/x509').keys()
-    ['./docs/x509/tutorial.rst', './docs/x509/reference.rst', './docs/x509/index.rst']
-    >>> statfiles('./docs/x509', relative='./').keys()
-    ['docs/x509/reference.rst', 'docs/x509/tutorial.rst', 'docs/x509/index.rst']
-    >>> statfiles('./docs/x509', relative='./docs/x509/').keys()
-    ['index.rst', 'tutorial.rst', 'reference.rst']
     '''
-    statinfo = dict()
+    info = dict()
     if not os.path.exists(name):
-        return statinfo
+        return info
     if not os.path.isdir(name):
         if relative:
             relpath = os.path.relpath(name, start=relative)
         else:
             relpath = name
-        statinfo[relpath] = statfile(name)
-        if statinfo[relpath] is None:
-            del statinfo[relpath]
+        info[relpath] = func(name)
+        if info[relpath] is None:
+            del info[relpath]
     else:
         for root, dirs, files in os.walk(name):
             inodes = list()
@@ -311,48 +341,16 @@ def statfiles(name, relative=None):
             inodes.extend(files)
             for x in inodes:
                 foundpath = os.path.join(root, x)
+                if os.path.isdir(foundpath):
+                    continue
                 if relative:
                     relpath = os.path.relpath(foundpath, start=relative)
                 else:
                     relpath = foundpath
-                statinfo[relpath] = statfile(foundpath)
-                if statinfo[relpath] is None:
-                    del statinfo[relpath]
-    return statinfo
-
-
-def att_statinfo(statinfo, attr='st_mtime', func=max):
-    if statinfo:
-        return func([getattr(v, attr) for v in statinfo.values()])
-    else:
-        return 0
-
-
-max_size = functools.partial(att_statinfo, attr='st_size', func=max)
-min_size = functools.partial(att_statinfo, attr='st_size', func=min)
-
-max_mtime = functools.partial(att_statinfo, attr='st_mtime', func=max)
-min_mtime = functools.partial(att_statinfo, attr='st_mtime', func=min)
-
-max_ctime = functools.partial(att_statinfo, attr='st_ctime', func=max)
-min_ctime = functools.partial(att_statinfo, attr='st_ctime', func=min)
-
-max_atime = functools.partial(att_statinfo, attr='st_atime', func=max)
-min_atime = functools.partial(att_statinfo, attr='st_atime', func=min)
-
-
-def sieve(operand, statinfo, attr='st_mtime', func=operator.gt):
-    result = set()
-    for fname, stbuf in statinfo.items():
-        if func(getattr(stbuf, attr), operand):
-            result.add(fname)
-    return result
-
-mtime_gt = functools.partial(sieve, attr='st_mtime', func=operator.gt)
-mtime_lt = functools.partial(sieve, attr='st_mtime', func=operator.lt)
-
-size_gt = functools.partial(sieve, attr='st_size', func=operator.gt)
-size_lt = functools.partial(sieve, attr='st_size', func=operator.lt)
+                info[relpath] = func(foundpath)
+                if info[relpath] is None:
+                    del info[relpath]
+    return info
 
 #
 # -- end of file

tox.ini

@@ -4,7 +4,8 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py27, py34
+envlist = py39, py310
+skip_missing_interpreters = True
 
 [testenv]
 commands = {envpython} setup.py test