Commit 08eea2e7 authored by Dan Allen's avatar Dan Allen

Merge issue-22-content-aggregator branch into master

resolves #22 content aggregator component

See merge request antora/antora-direct!13
parents 84422e45 ce9375a2
Pipeline #13111083 passed with stages
in 2 minutes and 49 seconds
{
"extends": "standard",
"rules": {
"max-len": ["error", {
"code": 120,
"ignoreStrings": true,
"ignoreUrls": true,
"ignoreTemplateLiterals": true
}],
"arrow-parens": ["error", "always"],
"comma-dangle": ["error", "always-multiline"]
}
......
......@@ -7,11 +7,7 @@ const format = require('./tasks/format-task')
const test = require('./tasks/test-task')
const testFiles = ['test/**/*-test.js', 'packages/*/test/**/*-test.js']
const allFiles = [
'gulpfile.js',
'{lib*,tasks,test}/**/*.js',
'packages/*/{lib,test}/**/*.js',
]
const allFiles = ['gulpfile.js', '{lib*,tasks,test}/**/*.js', 'packages/*/{lib,test}/**/*.js']
gulp.task('lint', () => lint(allFiles))
gulp.task('format', () => format(allFiles))
......
'use strict'
const path = require('path')
const fs = require('fs')
const _ = require('lodash')
const del = require('del')
const File = require('vinyl')
const git = require('nodegit')
const isMatch = require('matcher').isMatch
const mime = require('./mime')
const streamToArray = require('stream-to-array')
const vfs = require('vinyl-fs')
const yaml = require('js-yaml')
const localCachePath = path.resolve('.git-cache')
module.exports = async (playbook) => {
const componentVersions = playbook.content.sources.map(async (repo) => {
const { repository, isLocalRepo, isBare, url } = await openOrCloneRepository(repo.location)
const branches = await repository.getReferences(git.Reference.TYPE.OID)
const repoComponentVersions = _(branches)
.map((branch) => getBranchInfo(branch))
.groupBy('branchName')
.mapValues((unorderedBranches) => {
// isLocal comes from reference.isBranch() which is 0 or 1
// so we'll end up with truthy isLocal last in the array
const branches = _.sortBy(unorderedBranches, 'isLocal')
return isLocalRepo ? _.last(branches) : _.first(branches)
})
.values()
.filter(({ branchName }) => branchMatches(branchName, repo.branches))
.map(async ({ branch, branchName, isHead, isLocal }) => {
let files
if (isLocalRepo && !isBare && isHead) {
files = await loadLocalFiles(repo)
} else {
files = await loadGitFiles(repository, branch, repo)
}
const componentVersion = await readComponentDesc(files)
componentVersion.files = files.map((file) => assignFileProperties(file, url, branchName, repo.subpath))
return componentVersion
})
.value()
const allRepoComponentVersions = await Promise.all(repoComponentVersions)
// nodegit repositories need to be manually closed
await repository.free()
return allRepoComponentVersions
})
return buildCorpus(await Promise.all(componentVersions))
}
async function openOrCloneRepository (repoUrl) {
const isLocalRepo = isLocalDirectory(repoUrl)
let localPath
let repository
let isBare
if (isLocalRepo) {
localPath = repoUrl
isBare = !isLocalDirectory(path.join(localPath, '.git'))
} else {
localPath = localCachePath + '/' + repoUrl.replace(/[:/\\]+/g, '__')
isBare = true
}
try {
if (isBare) {
repository = await git.Repository.openBare(localPath)
if (!isLocalRepo) {
// fetches new branches and deletes old local ones
await repository.fetch('origin', Object.assign({ prune: 1 }, getFetchOptions()))
}
} else {
repository = await git.Repository.open(localPath)
}
} catch (e) {
if (!isLocalRepo) {
del.sync(localPath)
repository = await git.Clone.clone(repoUrl, localPath, {
bare: 1,
fetchOpts: getFetchOptions(),
})
}
}
let url
try {
const remoteObject = await repository.getRemote('origin')
url = remoteObject.url()
} catch (e) {
url = repoUrl
}
return { repository, isLocalRepo, isBare, url }
}
function isLocalDirectory (repoUrl) {
try {
const stats = fs.lstatSync(repoUrl)
return stats.isDirectory()
} catch (e) {
return false
}
}
function getFetchOptions () {
let sshKeyAuthAttempted
return {
callbacks: {
// https://github.com/nodegit/nodegit/blob/master/guides/cloning/ssh-with-agent/README.md#github-certificate-issue-in-os-x
certificateCheck: () => 1,
credentials: (_, username) => {
if (sshKeyAuthAttempted) {
throw new Error('Failed to authenticate git client using SSH key; SSH agent is not running')
} else {
sshKeyAuthAttempted = true
return git.Cred.sshKeyFromAgent(username)
}
},
},
}
}
function getBranchInfo (branch) {
const branchName = branch.shorthand().replace(/^origin\//, '')
const isLocal = branch.isBranch() === 1
const isHead = branch.isHead() === 1
return { branch, branchName, isLocal, isHead }
}
function branchMatches (branchName, branchPattern = '*') {
if (Array.isArray(branchPattern)) {
return branchPattern.some((pattern) => isMatch(branchName, pattern))
}
return isMatch(branchName, branchPattern)
}
function readComponentDesc (files) {
const componentDescFile = files.find((file) => file.relative === 'docs-component.yml')
if (componentDescFile == null) {
throw new Error('docs-component.yml not found')
}
const componentDesc = yaml.safeLoad(componentDescFile.contents.toString())
if (componentDesc.name == null) {
throw new Error('docs-component.yml is missing a name')
}
if (componentDesc.version == null) {
throw new Error('docs-component.yml is missing a version')
}
return componentDesc
}
async function loadGitFiles (repository, branch, repo) {
const tree = await getGitTree(repository, branch, repo.subpath)
const entries = await getGitEntries(tree)
const vfiles = entries.map(async (entry) => {
const blob = await entry.getBlob()
const contents = blob.content()
const stat = new fs.Stats({})
stat.mode = entry.filemode()
stat.size = contents.length
return new File({ path: entry.path(), contents, stat })
})
return Promise.all(vfiles)
}
async function getGitTree (repository, branch, subpath) {
const commit = await repository.getBranchCommit(branch)
const tree = await commit.getTree()
if (subpath == null) {
return tree
}
const subEntry = await tree.entryByPath(subpath)
const subTree = await repository.getTree(subEntry.id())
return subTree
}
function getGitEntries (tree, onEntry) {
return new Promise((resolve, reject) => {
const walker = tree.walk()
walker.on('error', (e) => reject(e))
walker.on('end', (entries) => resolve(entries))
walker.start()
})
}
async function loadLocalFiles (repo) {
const basePath = path.join(repo.location, repo.subpath || '.')
const vfileStream = vfs.src('**/*.*', {
base: basePath,
cwd: basePath,
dot: false,
})
return streamToArray(vfileStream)
}
function assignFileProperties (file, url, branch, subpath = '/') {
file.path = file.relative
file.base = process.cwd()
file.cwd = process.cwd()
const extname = path.extname(file.path)
file.src = {
basename: path.basename(file.path),
mediaType: mime.lookup(extname),
stem: path.basename(file.path, extname),
extname,
origin: {
git: { url, branch, subpath },
},
}
return file
}
function buildCorpus (componentVersions) {
return _(componentVersions)
.flatten()
.groupBy(({ name, version }) => `${version}@${name}`)
.map((componentVersions, id) => {
const component = _(componentVersions)
.map((a) => _.omit(a, 'files'))
.reduce((a, b) => _.assign(a, b), {})
component.files = _(componentVersions)
.map('files')
.reduce((a, b) => [...a, ...b], [])
return component
})
.sortBy(['name', 'version'])
.value()
}
'use strict'
const mime = require('mime-types')
mime.types['adoc'] = 'text/asciidoc'
mime.extensions['text/asciidoc'] = ['adoc']
module.exports = mime
:moduledir: ..
include::{moduledir}/_attributes.adoc[]
:moduledir: ..
include::{moduledir}/_attributes.adoc[]
= Page One
ifndef::env-site,env-github[]
include::_attributes.adoc[]
endif::[]
:keywords: foo, bar
Hey World!
= Page Two
ifndef::env-site,env-github[]
include::_attributes.adoc[]
endif::[]
:keywords: foo, bar
Bonjour!
= Page Three
ifndef::env-site,env-github[]
include::_attributes.adoc[]
endif::[]
:keywords: foo, bar
hello sample
= Page Four
ifndef::env-site,env-github[]
include::_attributes.adoc[]
endif::[]
:keywords: foo, bar
contents
{
"name": "foobar",
"version": "1.0.0",
"description": "desc"
}
This diff is collapsed.
'use strict'
const path = require('path')
const git = require('nodegit')
const fs = require('fs-extra')
const fixturesPath = path.resolve(__dirname, 'fixtures')
const reposBasePath = path.resolve(__dirname, 'repos')
class FixtureRepo {
constructor ({ isRemote, isBare }) {
this.isRemote = isRemote
this.isBare = isBare
}
async initRepo ({ repoName, name, title, version, nav, subpath }) {
this.subpath = subpath
this.repoPath = path.join(reposBasePath, repoName || name || 'default-repo')
this.location = this.repoPath
if (this.isRemote) {
this.location = 'file://' + this.location
}
if (this.isBare) {
this.location = this.location + '/.git'
}
this.repository = await git.Repository.init(this.repoPath, 0)
await this.copyAll(['README.adoc'])
await this.commitAll('Init commit', true)
await this.setDocsComponent({ name, title, version, nav, subpath })
return this
}
async copyAll (items, subpath = '.') {
return Promise.all(
items.map((item) => fs.copy(path.join(fixturesPath, item), path.join(this.repoPath, subpath, item)))
)
}
async removeAll (items) {
return Promise.all(items.map((item) => fs.remove(path.join(this.repoPath, item))))
}
async commitAll (message, firstCommit = false) {
const index = await this.repository.refreshIndex()
await index.addAll()
await index.write()
const oid = await index.writeTree()
const parentCommits = []
if (!firstCommit) {
const head = await git.Reference.nameToId(this.repository, 'HEAD')
const commit = await this.repository.getCommit(head)
parentCommits.push(commit)
}
return this.repository.createCommit(
'HEAD',
git.Signature.create('John Smith', 'john@smith.com', 123456789, 60),
git.Signature.create('John Smith', 'john@smith.com', 987654321, 90),
message,
oid,
parentCommits
)
}
async setDocsComponent ({ name, title, version, nav, subpath = '.' }) {
const filepath = path.join(this.repoPath, subpath, 'docs-component.yml')
const docsComponentYml = []
if (name) {
docsComponentYml.push(`name: ${name}`)
}
if (title) {
docsComponentYml.push(`title: ${title}`)
}
if (version) {
docsComponentYml.push(`version: '${version}'`)
}
if (nav) {
docsComponentYml.push('nav:')
nav.forEach((navItem) => {
docsComponentYml.push(` - ${navItem}`)
})
}
if (name != null || version != null) {
await fs.ensureFile(filepath)
await fs.writeFile(filepath, docsComponentYml.join('\n'))
await this.commitAll(`Set docs-component for ${version}`)
}
}
async createBranch ({ name, version, branch }) {
const branchName = branch || version
const head = await git.Reference.nameToId(this.repository, 'HEAD')
const commit = await this.repository.getCommit(head)
const branchReference = await this.repository.createBranch(branchName, commit, 0)
await this.repository.checkoutBranch(branchReference)
await this.setDocsComponent({ name, version })
}
async addFixtureFiles (files, subpath) {
await this.copyAll(files, subpath)
await this.commitAll('Add example files')
}
async removeFixtureFiles (files) {
await this.removeAll(files)
await this.commitAll('Remove example files')
}
}
module.exports = FixtureRepo
......@@ -8,8 +8,7 @@ module.exports = {
},
site: {
url: {
doc:
'The base URL of the published site (optional). Should not include a trailing slash.',
doc: 'The base URL of the published site (optional). Should not include a trailing slash.',
format: String,
default: undefined,
env: 'URL',
......@@ -22,20 +21,17 @@ module.exports = {
arg: 'title',
},
root: {
doc:
'The name of the component to use as the root of the site (optional).',
doc: 'The name of the component to use as the root of the site (optional).',
format: String,
default: undefined,
},
aspect: {
doc:
'The name of the aspect navigation to make available on every page in the site.',
doc: 'The name of the aspect navigation to make available on every page in the site.',
format: String,
default: undefined,
},
nav: {
doc:
'The list of descriptors which define the aspect navigation domains.',
doc: 'The list of descriptors which define the aspect navigation domains.',
format: Array,
default: undefined,
},
......@@ -79,15 +75,13 @@ module.exports = {
default: undefined,
},
archive: {
doc:
'A local theme archive. If specified, used in place of the UI bundle from the repository.',
doc: 'A local theme archive. If specified, used in place of the UI bundle from the repository.',
format: String,
default: undefined,
arg: 'ui-archive',
},
skip_cache: {
doc:
'Skip the local bundle cache and always fetch the UI bundle from the repository.',
doc: 'Skip the local bundle cache and always fetch the UI bundle from the repository.',
format: Boolean,
default: false,
arg: 'skip-ui-cache',
......@@ -109,23 +103,20 @@ module.exports = {
},
urls: {
htmlExtensionStyle: {
doc:
'Controls how the URL extension for HTML pages is handled (default, drop, or indexify).',
doc: 'Controls how the URL extension for HTML pages is handled (default, drop, or indexify).',
format: ['default', 'drop', 'indexify'],
default: 'default',
arg: 'html-url-extension-style',
},
aspectPageStrategy: {
doc:
'Controls how links to pages in aspect domains are generated (path or query).',
doc: 'Controls how links to pages in aspect domains are generated (path or query).',
format: String,
default: 'path',
arg: 'aspect-page-url-strategy',
},
},
redirects: {
doc:
'Generate nginx config file containing URL redirects for page aliases.',
doc: 'Generate nginx config file containing URL redirects for page aliases.',
format: Boolean,
default: false,
arg: 'redirects',
......
'use strict'
const fs = require('fs')
const path = require('path')
const convict = require('convict')
const cson = require('cson-parser')
const deepFreeze = require('deep-freeze')
const defaultSchema = require('./config/schema')
const yaml = require('js-yaml')
const cson = require('cson-parser')
function getConvictConfig (customSchema) {
if (customSchema != null) {
......
......@@ -60,10 +60,7 @@ describe('buildPlaybook()', () => {
},
two: 42,
three: false,
four: [
{ lastname: 'Lennon', name: 'John' },
{ lastname: 'McCartney', name: 'Paul' },
],
four: [{ lastname: 'Lennon', name: 'John' }, { lastname: 'McCartney', name: 'Paul' }],
}
})
......@@ -78,11 +75,7 @@ describe('buildPlaybook()', () => {
const csonSpec = path.resolve(__dirname, 'fixtures', 'spec-sample.cson')
const iniSpec = path.resolve(__dirname, 'fixtures', 'spec-sample.ini')
const badSpec = path.resolve(__dirname, 'fixtures', 'bad-spec-sample.yml')
const defaultSchemaSpec = path.resolve(
__dirname,
'fixtures',
'default-schema-spec-sample.yml'
)
const defaultSchemaSpec = path.resolve(__dirname, 'fixtures', 'default-schema-spec-sample.yml')
it('should throw error if no playbook spec file can be loaded', () => {
expect(() => buildPlaybook(schema)).to.throw()
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment