From a961647d66989aed2ccacbae82bb5860f2d5536b Mon Sep 17 00:00:00 2001 From: "Martin A. Brown" Date: Fri, 25 Mar 2016 11:46:04 -0700 Subject: [PATCH] a set of scripts for reorganizing the output tree --- LDP/migration-2016/faqmigration.py | 173 ++++++++++++++++++++ LDP/migration-2016/migration-helper.sh | 52 ++++-- LDP/migration-2016/migration-preparation.sh | 6 +- LDP/migration-2016/refmigration.py | 172 +++++++++++++++++++ 4 files changed, 389 insertions(+), 14 deletions(-) create mode 100644 LDP/migration-2016/faqmigration.py create mode 100644 LDP/migration-2016/refmigration.py diff --git a/LDP/migration-2016/faqmigration.py b/LDP/migration-2016/faqmigration.py new file mode 100644 index 00000000..ad68c0e5 --- /dev/null +++ b/LDP/migration-2016/faqmigration.py @@ -0,0 +1,173 @@ +#! /usr/bin/python +# +# -- migrate to the new naming scheme + +from __future__ import absolute_import, division, print_function + +import os +import sys +import time +import errno +import shutil +import logging +import functools + +logformat = '%(levelname)-9s %(name)s %(filename)s#%(lineno)s ' \ + + '%(funcName)s %(message)s' +logging.basicConfig(stream=sys.stderr, format=logformat, level=logging.DEBUG) +logger = logging.getLogger(__name__) + +# -- short names +# +opa = os.path.abspath +opb = os.path.basename +opd = os.path.dirname +opj = os.path.join +opn = os.path.normpath +opr = os.path.relpath +ops = os.path.split + +faqdocs = '''Ftape-FAQ +Linux-RAID-FAQ +LDP-FAQ +AfterStep-FAQ'''.split() + + +def validate_args(argv): + if len(argv) == 4: + for d in argv[:3]: + if not os.path.isdir(d): + return False + return True + return False + + +def collect_published_stems(dirbase): + d = dict() + for stem in os.listdir(dirbase): + if not os.path.isdir(opj(dirbase, stem)): + continue + d[stem] = stem + return d + + +def make_refresh(target, title, delay=0): + text = ''' + + {1}: {0} + + + +

This page has moved permanently to + {0}. + Update your bookmarks if you wish. The compatibility + redirect will remain through, at least, early 2017. +

+ + +''' + return text.format(target, title, delay) + +def swapfiles(a, b): + '''use os.rename() to make "a" become "b"''' + if not os.path.isfile(a): + raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), a) + tf = None + if os.path.exists(b): + _, tf = mkstemp(prefix='swapfile-', dir=opd(opa(a))) + logger.debug("Created tempfile %s.", tf) + logger.debug("About to rename %s to %s.", b, tf) + os.rename(b, tf) + logger.debug("About to rename %s to %s.", a, b) + os.rename(a, b) + if tf: + logger.debug("About to rename %s to %s.", tf, a) + os.rename(tf, a) + logger.debug("About to remove %s.", tf) + os.rmdir(tf) + + +def create_symlink(source, target): + assert not os.path.exists(target) + targetdir = os.path.dirname(target) + if not os.path.isdir(targetdir): + logger.debug("Creating directory %s", targetdir) + os.makedirs(targetdir) + logger.debug("Creating symlink %s, pointing to %s", target, source) + os.symlink(os.path.relpath(source, start=targetdir), target) + + +def create_refresh_meta_equiv(fname, url, stem, **kwargs): + assert not os.path.exists(fname) + targetdir = os.path.dirname(fname) + if not os.path.isdir(targetdir): + logger.debug("Creating directory %s", targetdir) + os.makedirs(targetdir) + logger.debug("Creating file %s, with redirect to %s", fname, url) + with open(fname, 'w') as f: + f.write(make_refresh(url, stem, **kwargs)) + + +def newhtmlfilename(pubdir, stem, fname): + sought = opj(pubdir, stem, fname) + if not os.path.isfile(sought): + return opj(pubdir, stem, 'index.html') + return sought + + +def faqs(stems, faqpath, faqcompat, pubdir, urlbase): + + for stem in faqdocs: + if stem not in stems: + logger.critical("Stem %s not found in %s.", stem, pubdir) + sys.exit(1) + + # -- PDF handling + newpdf = opj(pubdir, stem, stem + '.pdf') + oldpdf = opj(faqcompat, 'pdf', stem + '.pdf') + if os.path.exists(oldpdf): + assert os.path.exists(oldpdf) + assert os.path.exists(newpdf) + os.rename(oldpdf, oldpdf + '.' + str(int(time.time()))) + create_symlink(newpdf, oldpdf) + + # -- HTML handling + htmldir = opj(faqcompat, stem) + if os.path.isdir(htmldir): + # -- LDP-FAQ and Linux-RAID-FAQ + for fn in os.listdir(htmldir): + if not fn.endswith('.html'): + continue + pubpath = newhtmlfilename(pubdir, stem, fn) + url = pubpath.replace(pubdir, urlbase) + fullname = opj(htmldir, fn) + os.rename(fullname, fullname + '.' + str(int(time.time()))) + create_refresh_meta_equiv(fullname, url, stem, delay=2) + else: + # -- AfterStep-FAQ and Ftape-FAQ + htmldir = faqcompat + for fn in os.listdir(htmldir): + if not fn.startswith(stem): + continue + pubpath = newhtmlfilename(pubdir, stem, fn) + url = pubpath.replace(pubdir, urlbase) + fullname = opj(htmldir, fn) + os.rename(fullname, fullname + '.' + str(int(time.time()))) + create_refresh_meta_equiv(fullname, url, stem, delay=2) + + +def main(fin, fout, argv): + me = os.path.basename(sys.argv[0]) + usage = "usage: %s " % (me,) + if not validate_args(argv): + return usage + faqpath, faqcompat, pubdir, urlbase = argv + stems = collect_published_stems(pubdir) + faqs(stems, faqpath, faqcompat, pubdir, urlbase) + return os.EX_OK + + +if __name__ == '__main__': + sys.exit(main(sys.stdin, sys.stdout, sys.argv[1:])) + +# -- end of file diff --git a/LDP/migration-2016/migration-helper.sh b/LDP/migration-2016/migration-helper.sh index 8f44d346..26e79397 100644 --- a/LDP/migration-2016/migration-helper.sh +++ b/LDP/migration-2016/migration-helper.sh @@ -5,41 +5,67 @@ set -x SELFNAME="$( readlink --canonicalize ${0})" ME="${SELFNAME##*/}" # -- basename -DIR="${SELFNAME%/*}" # -- dirname - -HOWTO_MIGRATOR=${DIR}/howtomigration.py -GUIDE_MIGRATOR=${DIR}/guidemigration.py +HERE="${SELFNAME%/*}" # -- dirname +# -- SET THIS VARIABLE to the full path of the LDP content +# CONTENTROOT=/home/mabrown/wip/tldp/website/html -cd "$CONTENTROOT" # -- trailing slash, atypically included on PUBDIR, here PUBDIR="${CONTENTROOT}/en/" URL_PUBDIR=http://www.tldp.org/en/ -HOWTOS="${CONTENTROOT}/HOWTO" -GUIDES="${CONTENTROOT}/LDP" +cd "$CONTENTROOT" # -- HOWTO handling: build symlinks and HTTP META-EQUIV files # +HOWTOS="${CONTENTROOT}/HOWTO/" HOWTO_COMPAT=HOWTO.compat/ +HOWTO_MIGRATOR=${HERE}/howtomigration.py + test -d "${HOWTO_COMPAT}" \ || mkdir "${HOWTO_COMPAT}" - HOWTO_COMPAT=$( readlink --canonicalize "$HOWTO_COMPAT" ) - python \ "${HOWTO_MIGRATOR}" "${HOWTOS}" "${HOWTO_COMPAT}" "${PUBDIR}" "${URL_PUBDIR}" + +# -- guide handling: build symlinks and HTTP META-EQUIV files +# +GUIDES="${CONTENTROOT}/LDP/" GUIDE_COMPAT=LDP.compat/ -test -d "${GUIDE_COMPAT}" \ - || mkdir "${GUIDE_COMPAT}" - -rsync --archive --verbose ./LDP/ "${GUIDE_COMPAT}/" +GUIDE_MIGRATOR=${HERE}/guidemigration.py +rsync --archive --verbose -- "${GUIDES}" "${GUIDE_COMPAT}" +GUIDE_COMPAT=$( readlink --canonicalize "$GUIDE_COMPAT" ) python \ "${GUIDE_MIGRATOR}" "${GUIDES}" "${GUIDE_COMPAT}" "${PUBDIR}" "${URL_PUBDIR}" + +# -- ref handling: build symlinks and HTTP META-EQUIV files +# +REFS="${CONTENTROOT}/REF/" +REF_COMPAT=REF.compat/ +REF_MIGRATOR=${HERE}/refmigration.py + +rsync --archive --verbose -- "${REFS}" "${REF_COMPAT}" +REF_COMPAT=$( readlink --canonicalize "$REF_COMPAT" ) +python \ + "${REF_MIGRATOR}" "${REFS}" "${REF_COMPAT}" "${PUBDIR}" "${URL_PUBDIR}" + + +# -- ref handling: build symlinks and HTTP META-EQUIV files +# +FAQS="${CONTENTROOT}/FAQ/" +FAQ_COMPAT=FAQ.compat/ +FAQ_MIGRATOR=${HERE}/faqmigration.py + +rsync --archive --verbose -- "${FAQS}" "${FAQ_COMPAT}" +FAQ_COMPAT=$( readlink --canonicalize "$FAQ_COMPAT" ) +python \ + "${FAQ_MIGRATOR}" "${FAQS}" "${FAQ_COMPAT}" "${PUBDIR}" "${URL_PUBDIR}" + + exit 0 # -- end of file diff --git a/LDP/migration-2016/migration-preparation.sh b/LDP/migration-2016/migration-preparation.sh index 756231cd..52c6edac 100644 --- a/LDP/migration-2016/migration-preparation.sh +++ b/LDP/migration-2016/migration-preparation.sh @@ -41,8 +41,12 @@ mv \ REF/palmdevqs \ REF/ls_quickref \ REF/Joe-Command-Reference \ + FAQ/Linux-FAQ \ + FAQ/sig11 \ + FAQ/Threads-FAQ \ + FAQ/WordPerfect-Linux-FAQ \ -# -- and toss aside the really ancient crap +# -- and toss aside the neolithically-ancient crap # TODELETE=todelete-$( date +%F ) test -d "${TODELETE}" \ diff --git a/LDP/migration-2016/refmigration.py b/LDP/migration-2016/refmigration.py new file mode 100644 index 00000000..ffcfb8fe --- /dev/null +++ b/LDP/migration-2016/refmigration.py @@ -0,0 +1,172 @@ +#! /usr/bin/python +# +# -- migrate to the new naming scheme + +from __future__ import absolute_import, division, print_function + +import os +import sys +import time +import errno +import shutil +import logging +import functools + +logformat = '%(levelname)-9s %(name)s %(filename)s#%(lineno)s ' \ + + '%(funcName)s %(message)s' +logging.basicConfig(stream=sys.stderr, format=logformat, level=logging.DEBUG) +logger = logging.getLogger(__name__) + +# -- short names +# +opa = os.path.abspath +opb = os.path.basename +opd = os.path.dirname +opj = os.path.join +opn = os.path.normpath +opr = os.path.relpath +ops = os.path.split + +refdocs = '''CVS-BestPractices +VideoLAN-Quickstart +VLC-User-Guide +VLS-User-Guide +INTRO/Backup-INTRO +INTRO/Intrusion-INTRO +INTRO/PhysSecurity-INTRO +INTRO/SecuringData-INTRO +INTRO/Virus-INTRO'''.split() + + +def validate_args(argv): + if len(argv) == 4: + for d in argv[:3]: + if not os.path.isdir(d): + return False + return True + return False + + +def collect_published_stems(dirbase): + d = dict() + for stem in os.listdir(dirbase): + if not os.path.isdir(opj(dirbase, stem)): + continue + d[stem] = stem + # add_renamed_stems(d) + # add_skipped_stems(d) + return d + + +def make_refresh(target, title, delay=0): + text = ''' + + {1}: {0} + + + +

This page has moved permanently to + {0}. + Update your bookmarks if you wish. The compatibility + redirect will remain through, at least, early 2017. +

+ + +''' + return text.format(target, title, delay) + +def swapfiles(a, b): + '''use os.rename() to make "a" become "b"''' + if not os.path.isfile(a): + raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), a) + tf = None + if os.path.exists(b): + _, tf = mkstemp(prefix='swapfile-', dir=opd(opa(a))) + logger.debug("Created tempfile %s.", tf) + logger.debug("About to rename %s to %s.", b, tf) + os.rename(b, tf) + logger.debug("About to rename %s to %s.", a, b) + os.rename(a, b) + if tf: + logger.debug("About to rename %s to %s.", tf, a) + os.rename(tf, a) + logger.debug("About to remove %s.", tf) + os.rmdir(tf) + + +def create_symlink(source, target): + assert not os.path.exists(target) + targetdir = os.path.dirname(target) + if not os.path.isdir(targetdir): + logger.debug("Creating directory %s", targetdir) + os.makedirs(targetdir) + logger.debug("Creating symlink %s, pointing to %s", target, source) + os.symlink(os.path.relpath(source, start=targetdir), target) + + +def create_refresh_meta_equiv(fname, url, stem, **kwargs): + assert not os.path.exists(fname) + targetdir = os.path.dirname(fname) + if not os.path.isdir(targetdir): + logger.debug("Creating directory %s", targetdir) + os.makedirs(targetdir) + logger.debug("Creating file %s, with redirect to %s", fname, url) + with open(fname, 'w') as f: + f.write(make_refresh(url, stem, **kwargs)) + + +def newhtmlfilename(pubdir, stem, fname): + sought = opj(pubdir, stem, fname) + if not os.path.isfile(sought): + return opj(pubdir, stem, 'index.html') + return sought + + +def refs(stems, refpath, refcompat, pubdir, urlbase): + + for doc in refdocs: + stem = doc.replace('INTRO/', '') + if stem not in stems: + logger.critical("Stem %s not found in %s.", stem, pubdir) + sys.exit(1) + + # -- PDF handling + newpdf = opj(pubdir, stem, stem + '.pdf') + oldpdf = opj(refcompat, doc + '.pdf') + if not os.path.exists(oldpdf): + oldpdf = opj(refcompat, stem, stem + '.pdf') + assert os.path.exists(oldpdf) + assert os.path.exists(newpdf) + os.rename(oldpdf, oldpdf + '.' + str(int(time.time()))) + create_symlink(newpdf, oldpdf) + + # -- HTML handling + htmldir = opj(refcompat, doc, 'html') + if not os.path.isdir(htmldir): + htmldir, _ = os.path.split(htmldir) + assert os.path.exists(htmldir) + for fn in os.listdir(htmldir): + if not fn.endswith('.html'): + continue + pubpath = newhtmlfilename(pubdir, stem, fn) + url = pubpath.replace(pubdir, urlbase) + fullname = opj(htmldir, fn) + os.rename(fullname, fullname + '.' + str(int(time.time()))) + create_refresh_meta_equiv(fullname, url, stem, delay=2) + + +def main(fin, fout, argv): + me = os.path.basename(sys.argv[0]) + usage = "usage: %s " % (me,) + if not validate_args(argv): + return usage + refpath, refcompat, pubdir, urlbase = argv + stems = collect_published_stems(pubdir) + refs(stems, refpath, refcompat, pubdir, urlbase) + return os.EX_OK + + +if __name__ == '__main__': + sys.exit(main(sys.stdin, sys.stdout, sys.argv[1:])) + +# -- end of file