mirror of
https://github.com/vale981/sphinx-multiversion
synced 2025-03-05 09:31:40 -05:00
Refactor config loading and git tree parsing
This commit is contained in:
Jan Holthuis
parent
54112368be
commit
88171df2dd
3 changed files with 122 additions and 107 deletions
|
|
@ -5,23 +5,20 @@ import subprocess
|
|||
import re
|
||||
import tarfile
|
||||
|
||||
from . import sphinx
|
||||
|
||||
VersionRef = collections.namedtuple('VersionRef', [
|
||||
GitRef = collections.namedtuple('VersionRef', [
|
||||
'name',
|
||||
'commit',
|
||||
'source',
|
||||
'is_remote',
|
||||
'refname',
|
||||
'version',
|
||||
'release',
|
||||
])
|
||||
|
||||
|
||||
def get_refs(gitroot):
|
||||
def get_all_refs(gitroot):
|
||||
cmd = ("git", "for-each-ref", "--format", "%(objectname) %(refname)", "refs")
|
||||
output = subprocess.check_output(cmd, cwd=gitroot).decode()
|
||||
for line in output.splitlines():
|
||||
is_remote = False
|
||||
line = line.strip()
|
||||
# Parse refname
|
||||
matchobj = re.match(r"^(\w+) refs/(heads|tags|remotes/[^/]+)/(\S+)$", line)
|
||||
|
|
@ -31,40 +28,30 @@ def get_refs(gitroot):
|
|||
source = matchobj.group(2)
|
||||
name = matchobj.group(3)
|
||||
refname = line.partition(' ')[2]
|
||||
|
||||
yield (name, commit, source, refname)
|
||||
|
||||
|
||||
def get_conf(gitroot, refname, confpath):
|
||||
objectname = "{}:{}".format(refname, confpath)
|
||||
cmd = ("git", "show", objectname)
|
||||
return subprocess.check_output(cmd, cwd=gitroot).decode()
|
||||
|
||||
|
||||
def find_versions(gitroot, confpath, tag_whitelist, branch_whitelist, remote_whitelist):
|
||||
for name, commit, source, refname in get_refs(gitroot):
|
||||
is_remote = False
|
||||
if source == 'tags':
|
||||
if tag_whitelist is None or not re.match(tag_whitelist, name):
|
||||
continue
|
||||
elif source == 'heads':
|
||||
if branch_whitelist is None or not re.match(branch_whitelist, name):
|
||||
continue
|
||||
elif source.startswith('remotes/') and remote_whitelist is not None:
|
||||
if source.startswith('remotes/'):
|
||||
is_remote = True
|
||||
remote_name = source.partition('/')[2]
|
||||
|
||||
yield GitRef(name, commit, source, is_remote, refname)
|
||||
|
||||
|
||||
def get_refs(gitroot, tag_whitelist, branch_whitelist, remote_whitelist):
|
||||
for ref in get_all_refs(gitroot):
|
||||
if ref.source == 'tags':
|
||||
if tag_whitelist is None or not re.match(tag_whitelist, ref.name):
|
||||
continue
|
||||
elif ref.source == 'heads':
|
||||
if branch_whitelist is None or not re.match(branch_whitelist, ref.name):
|
||||
continue
|
||||
elif ref.is_remote and remote_whitelist is not None:
|
||||
remote_name = ref.source.partition('/')[2]
|
||||
if not re.match(remote_whitelist, remote_name):
|
||||
continue
|
||||
if branch_whitelist is None or not re.match(branch_whitelist, name):
|
||||
if branch_whitelist is None or not re.match(branch_whitelist, ref.name):
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
|
||||
conf = get_conf(gitroot, refname, confpath)
|
||||
config = sphinx.parse_conf(conf)
|
||||
version = config['version']
|
||||
release = config['release']
|
||||
yield VersionRef(name, commit, source, is_remote, refname, version, release)
|
||||
yield ref
|
||||
|
||||
|
||||
def copy_tree(src, dst, reference, sourcepath='.'):
|
||||
|
|
|
|||
|
|
@ -1,13 +1,16 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import re
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
from sphinx.cmd import build as sphinx_build
|
||||
from sphinx import config as sphinx_config
|
||||
from sphinx import project as sphinx_project
|
||||
|
||||
from . import sphinx
|
||||
|
|
@ -18,89 +21,115 @@ def main(argv=None):
|
|||
if not argv:
|
||||
argv = sys.argv[1:]
|
||||
|
||||
parser = sphinx_build.get_parser()
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
# Find the indices
|
||||
srcdir_index = None
|
||||
outdir_index = None
|
||||
for i, value in enumerate(argv):
|
||||
if value == args.sourcedir:
|
||||
argv[i] = '{{{SOURCEDIR}}}'
|
||||
test_args = parser.parse_args(argv)
|
||||
if test_args.sourcedir == argv[i]:
|
||||
srcdir_index = i
|
||||
argv[i] = args.sourcedir
|
||||
|
||||
if value == args.outputdir:
|
||||
argv[i] = '{{{OUTPUTDIR}}}'
|
||||
test_args = parser.parse_args(argv)
|
||||
if test_args.outputdir == argv[i]:
|
||||
outdir_index = i
|
||||
argv[i] = args.outputdir
|
||||
|
||||
if srcdir_index is None:
|
||||
raise ValueError("Failed to find srcdir index")
|
||||
if outdir_index is None:
|
||||
raise ValueError("Failed to find outdir index")
|
||||
|
||||
# Parse config
|
||||
confpath = os.path.join(args.confdir, 'conf.py')
|
||||
with open(confpath, mode='r') as f:
|
||||
config = sphinx.parse_conf(f.read())
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('sourcedir', help='path to documentation source files')
|
||||
parser.add_argument('outputdir', help='path to output directory')
|
||||
parser.add_argument('filenames', nargs='*', help='a list of specific files to rebuild. Ignored if -a is specified')
|
||||
parser.add_argument('-c', metavar='PATH', dest='confdir', help='path where configuration file (conf.py) is located (default: same as SOURCEDIR)')
|
||||
parser.add_argument('-C', action='store_true', dest='noconfig', help='use no config file at all, only -D options')
|
||||
parser.add_argument('-D', metavar='setting=value', action='append', dest='define', default=[], help='override a setting in configuration file')
|
||||
parser.add_argument('--dump-metadata', action='store_true', help='dump generated metadata and exit')
|
||||
args, argv = parser.parse_known_args(argv)
|
||||
if args.noconfig:
|
||||
return 1
|
||||
|
||||
# Conf-overrides
|
||||
confoverrides = {}
|
||||
for d in args.define:
|
||||
key, _, value = d.partition('=')
|
||||
config[key] = value
|
||||
confoverrides[key] = value
|
||||
|
||||
tag_whitelist = config.get('smv_tag_whitelist', sphinx.DEFAULT_TAG_WHITELIST)
|
||||
branch_whitelist = config.get('smv_branch_whitelist', sphinx.DEFAULT_BRANCH_WHITELIST)
|
||||
remote_whitelist = config.get('smv_remote_whitelist', sphinx.DEFAULT_REMOTE_WHITELIST)
|
||||
released_pattern = config.get('smv_released_pattern', sphinx.DEFAULT_RELEASED_PATTERN)
|
||||
outputdir_format = config.get('smv_outputdir_format', sphinx.DEFAULT_OUTPUTDIR_FORMAT)
|
||||
# Parse config
|
||||
config = sphinx_config.Config.read(
|
||||
os.path.abspath(args.confdir if args.confdir else args.sourcedir),
|
||||
confoverrides,
|
||||
)
|
||||
config.add("smv_tag_whitelist", sphinx.DEFAULT_TAG_WHITELIST, "html", str)
|
||||
config.add("smv_branch_whitelist", sphinx.DEFAULT_TAG_WHITELIST, "html", str)
|
||||
config.add("smv_remote_whitelist", sphinx.DEFAULT_REMOTE_WHITELIST, "html", str)
|
||||
config.add("smv_released_pattern", sphinx.DEFAULT_RELEASED_PATTERN, "html", str)
|
||||
config.add("smv_outputdir_format", sphinx.DEFAULT_OUTPUTDIR_FORMAT, "html", str)
|
||||
|
||||
# Get git references
|
||||
gitroot = pathlib.Path('.').resolve()
|
||||
versions = git.find_versions(str(gitroot), 'source/conf.py', tag_whitelist, branch_whitelist, remote_whitelist)
|
||||
gitrefs = git.get_refs(
|
||||
str(gitroot),
|
||||
config.smv_tag_whitelist,
|
||||
config.smv_branch_whitelist,
|
||||
config.smv_remote_whitelist,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Get Sourcedir
|
||||
sourcedir = os.path.relpath(args.sourcedir, str(gitroot))
|
||||
if args.confdir:
|
||||
confdir = os.path.relpath(args.confdir, str(gitroot))
|
||||
else:
|
||||
confdir = sourcedir
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# Generate Metadata
|
||||
metadata = {}
|
||||
outputdirs = set()
|
||||
sourcedir = os.path.relpath(args.sourcedir, str(gitroot))
|
||||
for versionref in versions:
|
||||
for gitref in gitrefs:
|
||||
# Clone Git repo
|
||||
repopath = os.path.join(tmp, gitref.commit)
|
||||
try:
|
||||
git.copy_tree(gitroot.as_uri(), repopath, gitref)
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
logger.error(
|
||||
"Failed to copy git tree for %s to %s",
|
||||
gitref.refname, repopath)
|
||||
continue
|
||||
|
||||
# Find config
|
||||
confpath = os.path.join(repopath, confdir)
|
||||
try:
|
||||
current_config = sphinx_config.Config.read(
|
||||
confpath,
|
||||
confoverrides,
|
||||
)
|
||||
except sphinx_config.ConfigError:
|
||||
logger.error(
|
||||
"Failed load config for %s from %s",
|
||||
gitref.refname, confpath)
|
||||
continue
|
||||
|
||||
# Ensure that there are not duplicate output dirs
|
||||
outputdir = sphinx.format_outputdir(
|
||||
outputdir_format, versionref, language=config["language"])
|
||||
outputdir = config.smv_outputdir_format.format(
|
||||
ref=gitref,
|
||||
config=current_config,
|
||||
)
|
||||
if outputdir in outputdirs:
|
||||
print("outputdir '%s' of version %r conflicts with other versions!"
|
||||
% (outputdir, versionref))
|
||||
logger.warning(
|
||||
"outputdir '%s' for %s conflicts with other versions",
|
||||
outputdir, gitref.name)
|
||||
continue
|
||||
outputdirs.add(outputdir)
|
||||
|
||||
# Clone Git repo
|
||||
repopath = os.path.join(tmp, str(hash(versionref)))
|
||||
srcdir = os.path.join(repopath, sourcedir)
|
||||
try:
|
||||
git.copy_tree(gitroot.as_uri(), repopath, versionref)
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
outputdirs.remove(outputdir)
|
||||
continue
|
||||
|
||||
# Get List of files
|
||||
source_suffixes = config.get("source_suffix", "")
|
||||
source_suffixes = current_config.source_suffix
|
||||
if isinstance(source_suffixes, str):
|
||||
source_suffixes = [source_suffixes]
|
||||
project = sphinx_project.Project(srcdir, source_suffixes)
|
||||
metadata[versionref.name] = {
|
||||
"name": versionref.name,
|
||||
"version": versionref.version,
|
||||
"release": versionref.release,
|
||||
"is_released": bool(re.match(released_pattern, versionref.refname)),
|
||||
"source": versionref.source,
|
||||
"sourcedir": srcdir,
|
||||
source_suffixes = [current_config.source_suffix]
|
||||
project = sphinx_project.Project(sourcedir, source_suffixes)
|
||||
metadata[gitref.name] = {
|
||||
"name": gitref.name,
|
||||
"version": current_config.version,
|
||||
"release": current_config.release,
|
||||
"is_released": bool(
|
||||
re.match(config.smv_released_pattern, gitref.refname)),
|
||||
"source": gitref.source,
|
||||
"sourcedir": sourcedir,
|
||||
"outputdir": outputdir,
|
||||
"docnames": list(project.discover())
|
||||
}
|
||||
|
||||
if args.dump_metadata:
|
||||
print(json.dumps(metadata, indent=2))
|
||||
return
|
||||
|
||||
# Write Metadata
|
||||
metadata_path = os.path.abspath(os.path.join(tmp, "versions.json"))
|
||||
with open(metadata_path, mode='w') as fp:
|
||||
json.dump(metadata, fp, indent=2)
|
||||
|
|
@ -108,15 +137,18 @@ def main(argv=None):
|
|||
# Run Sphinx
|
||||
argv.extend(["-D", "smv_metadata_path={}".format(metadata_path)])
|
||||
for version_name, data in metadata.items():
|
||||
outdir = os.path.join(args.outputdir, data["outputdir"])
|
||||
os.makedirs(outdir, exist_ok=True)
|
||||
|
||||
current_argv = argv.copy()
|
||||
current_argv.extend([
|
||||
*args.define,
|
||||
"-D", "smv_current_version={}".format(version_name),
|
||||
"-c", args.confdir,
|
||||
data["sourcedir"],
|
||||
outdir,
|
||||
*args.filenames,
|
||||
])
|
||||
|
||||
outdir = os.path.join(args.outputdir, data["outputdir"])
|
||||
current_argv[srcdir_index] = data["sourcedir"]
|
||||
current_argv[outdir_index] = outdir
|
||||
os.makedirs(outdir, exist_ok=True)
|
||||
status = sphinx_build.build_main(current_argv)
|
||||
if status not in (0, None):
|
||||
break
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ DEFAULT_TAG_WHITELIST = r'^.*$'
|
|||
DEFAULT_BRANCH_WHITELIST = r'^.*$'
|
||||
DEFAULT_REMOTE_WHITELIST = None
|
||||
DEFAULT_RELEASED_PATTERN = r'^tags/.*$'
|
||||
DEFAULT_OUTPUTDIR_FORMAT = r'{version.version}/{language}'
|
||||
DEFAULT_OUTPUTDIR_FORMAT = r'{config.version}/{config.language}'
|
||||
|
||||
Version = collections.namedtuple('Version', [
|
||||
'name',
|
||||
|
|
@ -107,10 +107,6 @@ def parse_conf(config):
|
|||
return module
|
||||
|
||||
|
||||
def format_outputdir(fmt, versionref, language):
|
||||
return fmt.format(version=versionref, language=language)
|
||||
|
||||
|
||||
def html_page_context(app, pagename, templatename, context, doctree):
|
||||
versioninfo = VersionInfo(
|
||||
app, context, app.config.smv_metadata, app.config.smv_current_version)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue