a set of scripts for reorganizing the output tree

This commit is contained in:
Martin A. Brown 2016-03-25 11:46:04 -07:00
parent 50cc0774b1
commit a961647d66
4 changed files with 389 additions and 14 deletions

View File

@ -0,0 +1,173 @@
#! /usr/bin/python
#
# -- migrate to the new naming scheme
from __future__ import absolute_import, division, print_function
import os
import sys
import time
import errno
import shutil
import logging
import functools
logformat = '%(levelname)-9s %(name)s %(filename)s#%(lineno)s ' \
+ '%(funcName)s %(message)s'
logging.basicConfig(stream=sys.stderr, format=logformat, level=logging.DEBUG)
logger = logging.getLogger(__name__)
# -- short names
#
opa = os.path.abspath
opb = os.path.basename
opd = os.path.dirname
opj = os.path.join
opn = os.path.normpath
opr = os.path.relpath
ops = os.path.split
faqdocs = '''Ftape-FAQ
Linux-RAID-FAQ
LDP-FAQ
AfterStep-FAQ'''.split()
def validate_args(argv):
if len(argv) == 4:
for d in argv[:3]:
if not os.path.isdir(d):
return False
return True
return False
def collect_published_stems(dirbase):
d = dict()
for stem in os.listdir(dirbase):
if not os.path.isdir(opj(dirbase, stem)):
continue
d[stem] = stem
return d
def make_refresh(target, title, delay=0):
text = '''<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>{1}: {0}</title>
<meta http-equiv="refresh" content="{2};URL='{0}'" />
</head>
<body>
<p>This page has moved permanently to
<a href="{0}">{0}</a>.
Update your bookmarks if you wish. The compatibility
redirect will remain through, at least, early 2017.
</p>
</body>
</html>
'''
return text.format(target, title, delay)
def swapfiles(a, b):
'''use os.rename() to make "a" become "b"'''
if not os.path.isfile(a):
raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), a)
tf = None
if os.path.exists(b):
_, tf = mkstemp(prefix='swapfile-', dir=opd(opa(a)))
logger.debug("Created tempfile %s.", tf)
logger.debug("About to rename %s to %s.", b, tf)
os.rename(b, tf)
logger.debug("About to rename %s to %s.", a, b)
os.rename(a, b)
if tf:
logger.debug("About to rename %s to %s.", tf, a)
os.rename(tf, a)
logger.debug("About to remove %s.", tf)
os.rmdir(tf)
def create_symlink(source, target):
assert not os.path.exists(target)
targetdir = os.path.dirname(target)
if not os.path.isdir(targetdir):
logger.debug("Creating directory %s", targetdir)
os.makedirs(targetdir)
logger.debug("Creating symlink %s, pointing to %s", target, source)
os.symlink(os.path.relpath(source, start=targetdir), target)
def create_refresh_meta_equiv(fname, url, stem, **kwargs):
assert not os.path.exists(fname)
targetdir = os.path.dirname(fname)
if not os.path.isdir(targetdir):
logger.debug("Creating directory %s", targetdir)
os.makedirs(targetdir)
logger.debug("Creating file %s, with redirect to %s", fname, url)
with open(fname, 'w') as f:
f.write(make_refresh(url, stem, **kwargs))
def newhtmlfilename(pubdir, stem, fname):
sought = opj(pubdir, stem, fname)
if not os.path.isfile(sought):
return opj(pubdir, stem, 'index.html')
return sought
def faqs(stems, faqpath, faqcompat, pubdir, urlbase):
for stem in faqdocs:
if stem not in stems:
logger.critical("Stem %s not found in %s.", stem, pubdir)
sys.exit(1)
# -- PDF handling
newpdf = opj(pubdir, stem, stem + '.pdf')
oldpdf = opj(faqcompat, 'pdf', stem + '.pdf')
if os.path.exists(oldpdf):
assert os.path.exists(oldpdf)
assert os.path.exists(newpdf)
os.rename(oldpdf, oldpdf + '.' + str(int(time.time())))
create_symlink(newpdf, oldpdf)
# -- HTML handling
htmldir = opj(faqcompat, stem)
if os.path.isdir(htmldir):
# -- LDP-FAQ and Linux-RAID-FAQ
for fn in os.listdir(htmldir):
if not fn.endswith('.html'):
continue
pubpath = newhtmlfilename(pubdir, stem, fn)
url = pubpath.replace(pubdir, urlbase)
fullname = opj(htmldir, fn)
os.rename(fullname, fullname + '.' + str(int(time.time())))
create_refresh_meta_equiv(fullname, url, stem, delay=2)
else:
# -- AfterStep-FAQ and Ftape-FAQ
htmldir = faqcompat
for fn in os.listdir(htmldir):
if not fn.startswith(stem):
continue
pubpath = newhtmlfilename(pubdir, stem, fn)
url = pubpath.replace(pubdir, urlbase)
fullname = opj(htmldir, fn)
os.rename(fullname, fullname + '.' + str(int(time.time())))
create_refresh_meta_equiv(fullname, url, stem, delay=2)
def main(fin, fout, argv):
me = os.path.basename(sys.argv[0])
usage = "usage: %s <faqpath> <faqcompat> <pubdir> <urlbase>" % (me,)
if not validate_args(argv):
return usage
faqpath, faqcompat, pubdir, urlbase = argv
stems = collect_published_stems(pubdir)
faqs(stems, faqpath, faqcompat, pubdir, urlbase)
return os.EX_OK
if __name__ == '__main__':
sys.exit(main(sys.stdin, sys.stdout, sys.argv[1:]))
# -- end of file

View File

@ -5,41 +5,67 @@ set -x
SELFNAME="$( readlink --canonicalize ${0})"
ME="${SELFNAME##*/}" # -- basename
DIR="${SELFNAME%/*}" # -- dirname
HOWTO_MIGRATOR=${DIR}/howtomigration.py
GUIDE_MIGRATOR=${DIR}/guidemigration.py
HERE="${SELFNAME%/*}" # -- dirname
# -- SET THIS VARIABLE to the full path of the LDP content
#
CONTENTROOT=/home/mabrown/wip/tldp/website/html
cd "$CONTENTROOT"
# -- trailing slash, atypically included on PUBDIR, here
PUBDIR="${CONTENTROOT}/en/"
URL_PUBDIR=http://www.tldp.org/en/
HOWTOS="${CONTENTROOT}/HOWTO"
GUIDES="${CONTENTROOT}/LDP"
cd "$CONTENTROOT"
# -- HOWTO handling: build symlinks and HTTP META-EQUIV files
#
HOWTOS="${CONTENTROOT}/HOWTO/"
HOWTO_COMPAT=HOWTO.compat/
HOWTO_MIGRATOR=${HERE}/howtomigration.py
test -d "${HOWTO_COMPAT}" \
|| mkdir "${HOWTO_COMPAT}"
HOWTO_COMPAT=$( readlink --canonicalize "$HOWTO_COMPAT" )
python \
"${HOWTO_MIGRATOR}" "${HOWTOS}" "${HOWTO_COMPAT}" "${PUBDIR}" "${URL_PUBDIR}"
# -- guide handling: build symlinks and HTTP META-EQUIV files
#
GUIDES="${CONTENTROOT}/LDP/"
GUIDE_COMPAT=LDP.compat/
test -d "${GUIDE_COMPAT}" \
|| mkdir "${GUIDE_COMPAT}"
rsync --archive --verbose ./LDP/ "${GUIDE_COMPAT}/"
GUIDE_MIGRATOR=${HERE}/guidemigration.py
rsync --archive --verbose -- "${GUIDES}" "${GUIDE_COMPAT}"
GUIDE_COMPAT=$( readlink --canonicalize "$GUIDE_COMPAT" )
python \
"${GUIDE_MIGRATOR}" "${GUIDES}" "${GUIDE_COMPAT}" "${PUBDIR}" "${URL_PUBDIR}"
# -- ref handling: build symlinks and HTTP META-EQUIV files
#
REFS="${CONTENTROOT}/REF/"
REF_COMPAT=REF.compat/
REF_MIGRATOR=${HERE}/refmigration.py
rsync --archive --verbose -- "${REFS}" "${REF_COMPAT}"
REF_COMPAT=$( readlink --canonicalize "$REF_COMPAT" )
python \
"${REF_MIGRATOR}" "${REFS}" "${REF_COMPAT}" "${PUBDIR}" "${URL_PUBDIR}"
# -- ref handling: build symlinks and HTTP META-EQUIV files
#
FAQS="${CONTENTROOT}/FAQ/"
FAQ_COMPAT=FAQ.compat/
FAQ_MIGRATOR=${HERE}/faqmigration.py
rsync --archive --verbose -- "${FAQS}" "${FAQ_COMPAT}"
FAQ_COMPAT=$( readlink --canonicalize "$FAQ_COMPAT" )
python \
"${FAQ_MIGRATOR}" "${FAQS}" "${FAQ_COMPAT}" "${PUBDIR}" "${URL_PUBDIR}"
exit 0
# -- end of file

View File

@ -41,8 +41,12 @@ mv \
REF/palmdevqs \
REF/ls_quickref \
REF/Joe-Command-Reference \
FAQ/Linux-FAQ \
FAQ/sig11 \
FAQ/Threads-FAQ \
FAQ/WordPerfect-Linux-FAQ \
# -- and toss aside the really ancient crap
# -- and toss aside the neolithically-ancient crap
#
TODELETE=todelete-$( date +%F )
test -d "${TODELETE}" \

View File

@ -0,0 +1,172 @@
#! /usr/bin/python
#
# -- migrate to the new naming scheme
from __future__ import absolute_import, division, print_function
import os
import sys
import time
import errno
import shutil
import logging
import functools
logformat = '%(levelname)-9s %(name)s %(filename)s#%(lineno)s ' \
+ '%(funcName)s %(message)s'
logging.basicConfig(stream=sys.stderr, format=logformat, level=logging.DEBUG)
logger = logging.getLogger(__name__)
# -- short names
#
opa = os.path.abspath
opb = os.path.basename
opd = os.path.dirname
opj = os.path.join
opn = os.path.normpath
opr = os.path.relpath
ops = os.path.split
refdocs = '''CVS-BestPractices
VideoLAN-Quickstart
VLC-User-Guide
VLS-User-Guide
INTRO/Backup-INTRO
INTRO/Intrusion-INTRO
INTRO/PhysSecurity-INTRO
INTRO/SecuringData-INTRO
INTRO/Virus-INTRO'''.split()
def validate_args(argv):
if len(argv) == 4:
for d in argv[:3]:
if not os.path.isdir(d):
return False
return True
return False
def collect_published_stems(dirbase):
d = dict()
for stem in os.listdir(dirbase):
if not os.path.isdir(opj(dirbase, stem)):
continue
d[stem] = stem
# add_renamed_stems(d)
# add_skipped_stems(d)
return d
def make_refresh(target, title, delay=0):
text = '''<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>{1}: {0}</title>
<meta http-equiv="refresh" content="{2};URL='{0}'" />
</head>
<body>
<p>This page has moved permanently to
<a href="{0}">{0}</a>.
Update your bookmarks if you wish. The compatibility
redirect will remain through, at least, early 2017.
</p>
</body>
</html>
'''
return text.format(target, title, delay)
def swapfiles(a, b):
'''use os.rename() to make "a" become "b"'''
if not os.path.isfile(a):
raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), a)
tf = None
if os.path.exists(b):
_, tf = mkstemp(prefix='swapfile-', dir=opd(opa(a)))
logger.debug("Created tempfile %s.", tf)
logger.debug("About to rename %s to %s.", b, tf)
os.rename(b, tf)
logger.debug("About to rename %s to %s.", a, b)
os.rename(a, b)
if tf:
logger.debug("About to rename %s to %s.", tf, a)
os.rename(tf, a)
logger.debug("About to remove %s.", tf)
os.rmdir(tf)
def create_symlink(source, target):
assert not os.path.exists(target)
targetdir = os.path.dirname(target)
if not os.path.isdir(targetdir):
logger.debug("Creating directory %s", targetdir)
os.makedirs(targetdir)
logger.debug("Creating symlink %s, pointing to %s", target, source)
os.symlink(os.path.relpath(source, start=targetdir), target)
def create_refresh_meta_equiv(fname, url, stem, **kwargs):
assert not os.path.exists(fname)
targetdir = os.path.dirname(fname)
if not os.path.isdir(targetdir):
logger.debug("Creating directory %s", targetdir)
os.makedirs(targetdir)
logger.debug("Creating file %s, with redirect to %s", fname, url)
with open(fname, 'w') as f:
f.write(make_refresh(url, stem, **kwargs))
def newhtmlfilename(pubdir, stem, fname):
sought = opj(pubdir, stem, fname)
if not os.path.isfile(sought):
return opj(pubdir, stem, 'index.html')
return sought
def refs(stems, refpath, refcompat, pubdir, urlbase):
for doc in refdocs:
stem = doc.replace('INTRO/', '')
if stem not in stems:
logger.critical("Stem %s not found in %s.", stem, pubdir)
sys.exit(1)
# -- PDF handling
newpdf = opj(pubdir, stem, stem + '.pdf')
oldpdf = opj(refcompat, doc + '.pdf')
if not os.path.exists(oldpdf):
oldpdf = opj(refcompat, stem, stem + '.pdf')
assert os.path.exists(oldpdf)
assert os.path.exists(newpdf)
os.rename(oldpdf, oldpdf + '.' + str(int(time.time())))
create_symlink(newpdf, oldpdf)
# -- HTML handling
htmldir = opj(refcompat, doc, 'html')
if not os.path.isdir(htmldir):
htmldir, _ = os.path.split(htmldir)
assert os.path.exists(htmldir)
for fn in os.listdir(htmldir):
if not fn.endswith('.html'):
continue
pubpath = newhtmlfilename(pubdir, stem, fn)
url = pubpath.replace(pubdir, urlbase)
fullname = opj(htmldir, fn)
os.rename(fullname, fullname + '.' + str(int(time.time())))
create_refresh_meta_equiv(fullname, url, stem, delay=2)
def main(fin, fout, argv):
me = os.path.basename(sys.argv[0])
usage = "usage: %s <refpath> <refcompat> <pubdir> <urlbase>" % (me,)
if not validate_args(argv):
return usage
refpath, refcompat, pubdir, urlbase = argv
stems = collect_published_stems(pubdir)
refs(stems, refpath, refcompat, pubdir, urlbase)
return os.EX_OK
if __name__ == '__main__':
sys.exit(main(sys.stdin, sys.stdout, sys.argv[1:]))
# -- end of file