Commit 08eea2e7 authored by Dan Allen's avatar Dan Allen

Merge issue-22-content-aggregator branch into master

resolves #22 content aggregator component

See merge request antora/antora-direct!13
parents 84422e45 ce9375a2
Pipeline #13111083 passed with stages
in 2 minutes and 49 seconds
{ {
"extends": "standard", "extends": "standard",
"rules": { "rules": {
"max-len": ["error", {
"code": 120,
"ignoreStrings": true,
"ignoreUrls": true,
"ignoreTemplateLiterals": true
}],
"arrow-parens": ["error", "always"], "arrow-parens": ["error", "always"],
"comma-dangle": ["error", "always-multiline"] "comma-dangle": ["error", "always-multiline"]
} }
......
...@@ -7,11 +7,7 @@ const format = require('./tasks/format-task') ...@@ -7,11 +7,7 @@ const format = require('./tasks/format-task')
const test = require('./tasks/test-task') const test = require('./tasks/test-task')
const testFiles = ['test/**/*-test.js', 'packages/*/test/**/*-test.js'] const testFiles = ['test/**/*-test.js', 'packages/*/test/**/*-test.js']
const allFiles = [ const allFiles = ['gulpfile.js', '{lib*,tasks,test}/**/*.js', 'packages/*/{lib,test}/**/*.js']
'gulpfile.js',
'{lib*,tasks,test}/**/*.js',
'packages/*/{lib,test}/**/*.js',
]
gulp.task('lint', () => lint(allFiles)) gulp.task('lint', () => lint(allFiles))
gulp.task('format', () => format(allFiles)) gulp.task('format', () => format(allFiles))
......
'use strict'
const path = require('path')
const fs = require('fs')
const _ = require('lodash')
const del = require('del')
const File = require('vinyl')
const git = require('nodegit')
const isMatch = require('matcher').isMatch
const mime = require('./mime')
const streamToArray = require('stream-to-array')
const vfs = require('vinyl-fs')
const yaml = require('js-yaml')
const localCachePath = path.resolve('.git-cache')
module.exports = async (playbook) => {
const componentVersions = playbook.content.sources.map(async (repo) => {
const { repository, isLocalRepo, isBare, url } = await openOrCloneRepository(repo.location)
const branches = await repository.getReferences(git.Reference.TYPE.OID)
const repoComponentVersions = _(branches)
.map((branch) => getBranchInfo(branch))
.groupBy('branchName')
.mapValues((unorderedBranches) => {
// isLocal comes from reference.isBranch() which is 0 or 1
// so we'll end up with truthy isLocal last in the array
const branches = _.sortBy(unorderedBranches, 'isLocal')
return isLocalRepo ? _.last(branches) : _.first(branches)
})
.values()
.filter(({ branchName }) => branchMatches(branchName, repo.branches))
.map(async ({ branch, branchName, isHead, isLocal }) => {
let files
if (isLocalRepo && !isBare && isHead) {
files = await loadLocalFiles(repo)
} else {
files = await loadGitFiles(repository, branch, repo)
}
const componentVersion = await readComponentDesc(files)
componentVersion.files = files.map((file) => assignFileProperties(file, url, branchName, repo.subpath))
return componentVersion
})
.value()
const allRepoComponentVersions = await Promise.all(repoComponentVersions)
// nodegit repositories need to be manually closed
await repository.free()
return allRepoComponentVersions
})
return buildCorpus(await Promise.all(componentVersions))
}
async function openOrCloneRepository (repoUrl) {
const isLocalRepo = isLocalDirectory(repoUrl)
let localPath
let repository
let isBare
if (isLocalRepo) {
localPath = repoUrl
isBare = !isLocalDirectory(path.join(localPath, '.git'))
} else {
localPath = localCachePath + '/' + repoUrl.replace(/[:/\\]+/g, '__')
isBare = true
}
try {
if (isBare) {
repository = await git.Repository.openBare(localPath)
if (!isLocalRepo) {
// fetches new branches and deletes old local ones
await repository.fetch('origin', Object.assign({ prune: 1 }, getFetchOptions()))
}
} else {
repository = await git.Repository.open(localPath)
}
} catch (e) {
if (!isLocalRepo) {
del.sync(localPath)
repository = await git.Clone.clone(repoUrl, localPath, {
bare: 1,
fetchOpts: getFetchOptions(),
})
}
}
let url
try {
const remoteObject = await repository.getRemote('origin')
url = remoteObject.url()
} catch (e) {
url = repoUrl
}
return { repository, isLocalRepo, isBare, url }
}
function isLocalDirectory (repoUrl) {
try {
const stats = fs.lstatSync(repoUrl)
return stats.isDirectory()
} catch (e) {
return false
}
}
function getFetchOptions () {
let sshKeyAuthAttempted
return {
callbacks: {
// https://github.com/nodegit/nodegit/blob/master/guides/cloning/ssh-with-agent/README.md#github-certificate-issue-in-os-x
certificateCheck: () => 1,
credentials: (_, username) => {
if (sshKeyAuthAttempted) {
throw new Error('Failed to authenticate git client using SSH key; SSH agent is not running')
} else {
sshKeyAuthAttempted = true
return git.Cred.sshKeyFromAgent(username)
}
},
},
}
}
function getBranchInfo (branch) {
const branchName = branch.shorthand().replace(/^origin\//, '')
const isLocal = branch.isBranch() === 1
const isHead = branch.isHead() === 1
return { branch, branchName, isLocal, isHead }
}
function branchMatches (branchName, branchPattern = '*') {
if (Array.isArray(branchPattern)) {
return branchPattern.some((pattern) => isMatch(branchName, pattern))
}
return isMatch(branchName, branchPattern)
}
function readComponentDesc (files) {
const componentDescFile = files.find((file) => file.relative === 'docs-component.yml')
if (componentDescFile == null) {
throw new Error('docs-component.yml not found')
}
const componentDesc = yaml.safeLoad(componentDescFile.contents.toString())
if (componentDesc.name == null) {
throw new Error('docs-component.yml is missing a name')
}
if (componentDesc.version == null) {
throw new Error('docs-component.yml is missing a version')
}
return componentDesc
}
async function loadGitFiles (repository, branch, repo) {
const tree = await getGitTree(repository, branch, repo.subpath)
const entries = await getGitEntries(tree)
const vfiles = entries.map(async (entry) => {
const blob = await entry.getBlob()
const contents = blob.content()
const stat = new fs.Stats({})
stat.mode = entry.filemode()
stat.size = contents.length
return new File({ path: entry.path(), contents, stat })
})
return Promise.all(vfiles)
}
async function getGitTree (repository, branch, subpath) {
const commit = await repository.getBranchCommit(branch)
const tree = await commit.getTree()
if (subpath == null) {
return tree
}
const subEntry = await tree.entryByPath(subpath)
const subTree = await repository.getTree(subEntry.id())
return subTree
}
function getGitEntries (tree, onEntry) {
return new Promise((resolve, reject) => {
const walker = tree.walk()
walker.on('error', (e) => reject(e))
walker.on('end', (entries) => resolve(entries))
walker.start()
})
}
async function loadLocalFiles (repo) {
const basePath = path.join(repo.location, repo.subpath || '.')
const vfileStream = vfs.src('**/*.*', {
base: basePath,
cwd: basePath,
dot: false,
})
return streamToArray(vfileStream)
}
function assignFileProperties (file, url, branch, subpath = '/') {
file.path = file.relative
file.base = process.cwd()
file.cwd = process.cwd()
const extname = path.extname(file.path)
file.src = {
basename: path.basename(file.path),
mediaType: mime.lookup(extname),
stem: path.basename(file.path, extname),
extname,
origin: {
git: { url, branch, subpath },
},
}
return file
}
function buildCorpus (componentVersions) {
return _(componentVersions)
.flatten()
.groupBy(({ name, version }) => `${version}@${name}`)
.map((componentVersions, id) => {
const component = _(componentVersions)
.map((a) => _.omit(a, 'files'))
.reduce((a, b) => _.assign(a, b), {})
component.files = _(componentVersions)
.map('files')
.reduce((a, b) => [...a, ...b], [])
return component
})
.sortBy(['name', 'version'])
.value()
}
'use strict'
const mime = require('mime-types')
mime.types['adoc'] = 'text/asciidoc'
mime.extensions['text/asciidoc'] = ['adoc']
module.exports = mime
:moduledir: ..
include::{moduledir}/_attributes.adoc[]
:moduledir: ..
include::{moduledir}/_attributes.adoc[]
= Page One
ifndef::env-site,env-github[]
include::_attributes.adoc[]
endif::[]
:keywords: foo, bar
Hey World!
= Page Two
ifndef::env-site,env-github[]
include::_attributes.adoc[]
endif::[]
:keywords: foo, bar
Bonjour!
= Page Three
ifndef::env-site,env-github[]
include::_attributes.adoc[]
endif::[]
:keywords: foo, bar
hello sample
= Page Four
ifndef::env-site,env-github[]
include::_attributes.adoc[]
endif::[]
:keywords: foo, bar
contents
{
"name": "foobar",
"version": "1.0.0",
"description": "desc"
}
This diff is collapsed.
'use strict'
const path = require('path')
const git = require('nodegit')
const fs = require('fs-extra')
const fixturesPath = path.resolve(__dirname, 'fixtures')
const reposBasePath = path.resolve(__dirname, 'repos')
class FixtureRepo {
constructor ({ isRemote, isBare }) {
this.isRemote = isRemote
this.isBare = isBare
}
async initRepo ({ repoName, name, title, version, nav, subpath }) {
this.subpath = subpath
this.repoPath = path.join(reposBasePath, repoName || name || 'default-repo')
this.location = this.repoPath
if (this.isRemote) {
this.location = 'file://' + this.location
}
if (this.isBare) {
this.location = this.location + '/.git'
}
this.repository = await git.Repository.init(this.repoPath, 0)
await this.copyAll(['README.adoc'])
await this.commitAll('Init commit', true)
await this.setDocsComponent({ name, title, version, nav, subpath })
return this
}
async copyAll (items, subpath = '.') {
return Promise.all(
items.map((item) => fs.copy(path.join(fixturesPath, item), path.join(this.repoPath, subpath, item)))
)
}
async removeAll (items) {
return Promise.all(items.map((item) => fs.remove(path.join(this.repoPath, item))))
}
async commitAll (message, firstCommit = false) {
const index = await this.repository.refreshIndex()
await index.addAll()
await index.write()
const oid = await index.writeTree()
const parentCommits = []
if (!firstCommit) {
const head = await git.Reference.nameToId(this.repository, 'HEAD')
const commit = await this.repository.getCommit(head)
parentCommits.push(commit)
}
return this.repository.createCommit(
'HEAD',
git.Signature.create('John Smith', 'john@smith.com', 123456789, 60),
git.Signature.create('John Smith', 'john@smith.com', 987654321, 90),
message,
oid,
parentCommits
)
}
async setDocsComponent ({ name, title, version, nav, subpath = '.' }) {
const filepath = path.join(this.repoPath, subpath, 'docs-component.yml')
const docsComponentYml = []
if (name) {
docsComponentYml.push(`name: ${name}`)
}
if (title) {
docsComponentYml.push(`title: ${title}`)
}
if (version) {
docsComponentYml.push(`version: '${version}'`)
}
if (nav) {
docsComponentYml.push('nav:')
nav.forEach((navItem) => {
docsComponentYml.push(` - ${navItem}`)
})
}
if (name != null || version != null) {
await fs.ensureFile(filepath)
await fs.writeFile(filepath, docsComponentYml.join('\n'))
await this.commitAll(`Set docs-component for ${version}`)
}
}
async createBranch ({ name, version, branch }) {
const branchName = branch || version
const head = await git.Reference.nameToId(this.repository, 'HEAD')
const commit = await this.repository.getCommit(head)
const branchReference = await this.repository.createBranch(branchName, commit, 0)
await this.repository.checkoutBranch(branchReference)
await this.setDocsComponent({ name, version })
}
async addFixtureFiles (files, subpath) {
await this.copyAll(files, subpath)
await this.commitAll('Add example files')
}
async removeFixtureFiles (files) {
await this.removeAll(files)
await this.commitAll('Remove example files')
}
}
module.exports = FixtureRepo
...@@ -8,8 +8,7 @@ module.exports = { ...@@ -8,8 +8,7 @@ module.exports = {
}, },
site: { site: {
url: { url: {
doc: doc: 'The base URL of the published site (optional). Should not include a trailing slash.',
'The base URL of the published site (optional). Should not include a trailing slash.',
format: String, format: String,
default: undefined, default: undefined,
env: 'URL', env: 'URL',
...@@ -22,20 +21,17 @@ module.exports = { ...@@ -22,20 +21,17 @@ module.exports = {
arg: 'title', arg: 'title',
}, },
root: { root: {
doc: doc: 'The name of the component to use as the root of the site (optional).',
'The name of the component to use as the root of the site (optional).',
format: String, format: String,
default: undefined, default: undefined,
}, },
aspect: { aspect: {
doc: doc: 'The name of the aspect navigation to make available on every page in the site.',
'The name of the aspect navigation to make available on every page in the site.',
format: String, format: String,
default: undefined, default: undefined,
}, },
nav: { nav: {
doc: doc: 'The list of descriptors which define the aspect navigation domains.',
'The list of descriptors which define the aspect navigation domains.',
format: Array, format: Array,
default: undefined, default: undefined,
}, },
...@@ -79,15 +75,13 @@ module.exports = { ...@@ -79,15 +75,13 @@ module.exports = {
default: undefined, default: undefined,
}, },
archive: { archive: {
doc: doc: 'A local theme archive. If specified, used in place of the UI bundle from the repository.',
'A local theme archive. If specified, used in place of the UI bundle from the repository.',
format: String, format: String,
default: undefined, default: undefined,
arg: 'ui-archive', arg: 'ui-archive',
}, },
skip_cache: { skip_cache: {
doc: doc: 'Skip the local bundle cache and always fetch the UI bundle from the repository.',
'Skip the local bundle cache and always fetch the UI bundle from the repository.',
format: Boolean, format: Boolean,
default: false, default: false,
arg: 'skip-ui-cache', arg: 'skip-ui-cache',
...@@ -109,23 +103,20 @@ module.exports = { ...@@ -109,23 +103,20 @@ module.exports = {
}, },
urls: { urls: {
htmlExtensionStyle: { htmlExtensionStyle: {
doc: doc: 'Controls how the URL extension for HTML pages is handled (default, drop, or indexify).',
'Controls how the URL extension for HTML pages is handled (default, drop, or indexify).',
format: ['default', 'drop', 'indexify'], format: ['default', 'drop', 'indexify'],
default: 'default', default: 'default',
arg: 'html-url-extension-style', arg: 'html-url-extension-style',
}, },
aspectPageStrategy: { aspectPageStrategy: {
doc: doc: 'Controls how links to pages in aspect domains are generated (path or query).',
'Controls how links to pages in aspect domains are generated (path or query).',
format: String, format: String,
default: 'path', default: 'path',
arg: 'aspect-page-url-strategy', arg: 'aspect-page-url-strategy',
}, },
}, },
redirects: { redirects: {
doc: doc: 'Generate nginx config file containing URL redirects for page aliases.',
'Generate nginx config file containing URL redirects for page aliases.',
format: Boolean, format: Boolean,
default: false, default: false,
arg: 'redirects', arg: 'redirects',
......
'use strict'
const fs = require('fs') const fs = require('fs')
const path = require('path') const path = require('path')
const convict = require('convict') const convict = require('convict')
const cson = require('cson-parser')
const deepFreeze = require('deep-freeze') const deepFreeze = require('deep-freeze')
const defaultSchema = require('./config/schema') const defaultSchema = require('./config/schema')
const yaml = require('js-yaml') const yaml = require('js-yaml')
const cson = require('cson-parser')
function getConvictConfig (customSchema) { function getConvictConfig (customSchema) {
if (customSchema != null) { if (customSchema != null) {
......
...@@ -60,10 +60,7 @@ describe('buildPlaybook()', () => { ...@@ -60,10 +60,7 @@ describe('buildPlaybook()', () => {
}, },
two: 42, two: 42,
three: false, three: false,
four: [ four: [{ lastname: 'Lennon', name: 'John' }, { lastname: 'McCartney', name: 'Paul' }],
{ lastname: 'Lennon', name: 'John' },
{ lastname: 'McCartney', name: 'Paul' },
],
} }
}) })
...@@ -78,11 +75,7 @@ describe('buildPlaybook()', () => { ...@@ -78,11 +75,7 @@ describe('buildPlaybook()', () => {
const csonSpec = path.resolve(__dirname, 'fixtures', 'spec-sample.cson') const csonSpec = path.resolve(__dirname, 'fixtures', 'spec-sample.cson')
const iniSpec = path.resolve(__dirname, 'fixtures', 'spec-sample.ini') const iniSpec = path.resolve(__dirname, 'fixtures', 'spec-sample.ini')
const badSpec = path.resolve(__dirname, 'fixtures', 'bad-spec-sample.yml') const badSpec = path.resolve(__dirname, 'fixtures', 'bad-spec-sample.yml')
const defaultSchemaSpec = path.resolve( const defaultSchemaSpec = path.resolve(__dirname, 'fixtures', 'default-schema-spec-sample.yml')
__dirname,
'fixtures',
'default-schema-spec-sample.yml'
)
it('should throw error if no playbook spec file can be loaded', () => { it('should throw error if no playbook spec file can be loaded', () => {
expect(() => buildPlaybook(schema)).to.throw() expect(() => buildPlaybook(schema)).to.throw()
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment