Commit 08eea2e7 authored by Dan Allen's avatar Dan Allen

Merge issue-22-content-aggregator branch into master

resolves #22 content aggregator component

See merge request antora/antora-direct!13
parents 84422e45 ce9375a2
Pipeline #13111083 passed with stages
in 2 minutes and 49 seconds
{
"extends": "standard",
"rules": {
"max-len": ["error", {
"code": 120,
"ignoreStrings": true,
"ignoreUrls": true,
"ignoreTemplateLiterals": true
}],
"arrow-parens": ["error", "always"],
"comma-dangle": ["error", "always-multiline"]
}
......
......@@ -7,11 +7,7 @@ const format = require('./tasks/format-task')
const test = require('./tasks/test-task')
const testFiles = ['test/**/*-test.js', 'packages/*/test/**/*-test.js']
const allFiles = [
'gulpfile.js',
'{lib*,tasks,test}/**/*.js',
'packages/*/{lib,test}/**/*.js',
]
const allFiles = ['gulpfile.js', '{lib*,tasks,test}/**/*.js', 'packages/*/{lib,test}/**/*.js']
gulp.task('lint', () => lint(allFiles))
gulp.task('format', () => format(allFiles))
......
'use strict'
const path = require('path')
const fs = require('fs')
const _ = require('lodash')
const del = require('del')
const File = require('vinyl')
const git = require('nodegit')
const isMatch = require('matcher').isMatch
const mime = require('./mime')
const streamToArray = require('stream-to-array')
const vfs = require('vinyl-fs')
const yaml = require('js-yaml')
const localCachePath = path.resolve('.git-cache')
module.exports = async (playbook) => {
const componentVersions = playbook.content.sources.map(async (repo) => {
const { repository, isLocalRepo, isBare, url } = await openOrCloneRepository(repo.location)
const branches = await repository.getReferences(git.Reference.TYPE.OID)
const repoComponentVersions = _(branches)
.map((branch) => getBranchInfo(branch))
.groupBy('branchName')
.mapValues((unorderedBranches) => {
// isLocal comes from reference.isBranch() which is 0 or 1
// so we'll end up with truthy isLocal last in the array
const branches = _.sortBy(unorderedBranches, 'isLocal')
return isLocalRepo ? _.last(branches) : _.first(branches)
})
.values()
.filter(({ branchName }) => branchMatches(branchName, repo.branches))
.map(async ({ branch, branchName, isHead, isLocal }) => {
let files
if (isLocalRepo && !isBare && isHead) {
files = await loadLocalFiles(repo)
} else {
files = await loadGitFiles(repository, branch, repo)
}
const componentVersion = await readComponentDesc(files)
componentVersion.files = files.map((file) => assignFileProperties(file, url, branchName, repo.subpath))
return componentVersion
})
.value()
const allRepoComponentVersions = await Promise.all(repoComponentVersions)
// nodegit repositories need to be manually closed
await repository.free()
return allRepoComponentVersions
})
return buildCorpus(await Promise.all(componentVersions))
}
async function openOrCloneRepository (repoUrl) {
const isLocalRepo = isLocalDirectory(repoUrl)
let localPath
let repository
let isBare
if (isLocalRepo) {
localPath = repoUrl
isBare = !isLocalDirectory(path.join(localPath, '.git'))
} else {
localPath = localCachePath + '/' + repoUrl.replace(/[:/\\]+/g, '__')
isBare = true
}
try {
if (isBare) {
repository = await git.Repository.openBare(localPath)
if (!isLocalRepo) {
// fetches new branches and deletes old local ones
await repository.fetch('origin', Object.assign({ prune: 1 }, getFetchOptions()))
}
} else {
repository = await git.Repository.open(localPath)
}
} catch (e) {
if (!isLocalRepo) {
del.sync(localPath)
repository = await git.Clone.clone(repoUrl, localPath, {
bare: 1,
fetchOpts: getFetchOptions(),
})
}
}
let url
try {
const remoteObject = await repository.getRemote('origin')
url = remoteObject.url()
} catch (e) {
url = repoUrl
}
return { repository, isLocalRepo, isBare, url }
}
function isLocalDirectory (repoUrl) {
try {
const stats = fs.lstatSync(repoUrl)
return stats.isDirectory()
} catch (e) {
return false
}
}
function getFetchOptions () {
let sshKeyAuthAttempted
return {
callbacks: {
// https://github.com/nodegit/nodegit/blob/master/guides/cloning/ssh-with-agent/README.md#github-certificate-issue-in-os-x
certificateCheck: () => 1,
credentials: (_, username) => {
if (sshKeyAuthAttempted) {
throw new Error('Failed to authenticate git client using SSH key; SSH agent is not running')
} else {
sshKeyAuthAttempted = true
return git.Cred.sshKeyFromAgent(username)
}
},
},
}
}
function getBranchInfo (branch) {
const branchName = branch.shorthand().replace(/^origin\//, '')
const isLocal = branch.isBranch() === 1
const isHead = branch.isHead() === 1
return { branch, branchName, isLocal, isHead }
}
function branchMatches (branchName, branchPattern = '*') {
if (Array.isArray(branchPattern)) {
return branchPattern.some((pattern) => isMatch(branchName, pattern))
}
return isMatch(branchName, branchPattern)
}
function readComponentDesc (files) {
const componentDescFile = files.find((file) => file.relative === 'docs-component.yml')
if (componentDescFile == null) {
throw new Error('docs-component.yml not found')
}
const componentDesc = yaml.safeLoad(componentDescFile.contents.toString())
if (componentDesc.name == null) {
throw new Error('docs-component.yml is missing a name')
}
if (componentDesc.version == null) {
throw new Error('docs-component.yml is missing a version')
}
return componentDesc
}
async function loadGitFiles (repository, branch, repo) {
const tree = await getGitTree(repository, branch, repo.subpath)
const entries = await getGitEntries(tree)
const vfiles = entries.map(async (entry) => {
const blob = await entry.getBlob()
const contents = blob.content()
const stat = new fs.Stats({})
stat.mode = entry.filemode()
stat.size = contents.length
return new File({ path: entry.path(), contents, stat })
})
return Promise.all(vfiles)
}
async function getGitTree (repository, branch, subpath) {
const commit = await repository.getBranchCommit(branch)
const tree = await commit.getTree()
if (subpath == null) {
return tree
}
const subEntry = await tree.entryByPath(subpath)
const subTree = await repository.getTree(subEntry.id())
return subTree
}
function getGitEntries (tree, onEntry) {
return new Promise((resolve, reject) => {
const walker = tree.walk()
walker.on('error', (e) => reject(e))
walker.on('end', (entries) => resolve(entries))
walker.start()
})
}
async function loadLocalFiles (repo) {
const basePath = path.join(repo.location, repo.subpath || '.')
const vfileStream = vfs.src('**/*.*', {
base: basePath,
cwd: basePath,
dot: false,
})
return streamToArray(vfileStream)
}
function assignFileProperties (file, url, branch, subpath = '/') {
file.path = file.relative
file.base = process.cwd()
file.cwd = process.cwd()
const extname = path.extname(file.path)
file.src = {
basename: path.basename(file.path),
mediaType: mime.lookup(extname),
stem: path.basename(file.path, extname),
extname,
origin: {
git: { url, branch, subpath },
},
}
return file
}
function buildCorpus (componentVersions) {
return _(componentVersions)
.flatten()
.groupBy(({ name, version }) => `${version}@${name}`)
.map((componentVersions, id) => {
const component = _(componentVersions)
.map((a) => _.omit(a, 'files'))
.reduce((a, b) => _.assign(a, b), {})
component.files = _(componentVersions)
.map('files')
.reduce((a, b) => [...a, ...b], [])
return component
})
.sortBy(['name', 'version'])
.value()
}
'use strict'
const mime = require('mime-types')
mime.types['adoc'] = 'text/asciidoc'
mime.extensions['text/asciidoc'] = ['adoc']
module.exports = mime
:moduledir: ..
include::{moduledir}/_attributes.adoc[]
:moduledir: ..
include::{moduledir}/_attributes.adoc[]
= Page One
ifndef::env-site,env-github[]
include::_attributes.adoc[]
endif::[]
:keywords: foo, bar
Hey World!
= Page Two
ifndef::env-site,env-github[]
include::_attributes.adoc[]
endif::[]
:keywords: foo, bar
Bonjour!
= Page Three
ifndef::env-site,env-github[]
include::_attributes.adoc[]
endif::[]
:keywords: foo, bar
hello sample
= Page Four
ifndef::env-site,env-github[]
include::_attributes.adoc[]
endif::[]
:keywords: foo, bar
contents
{
"name": "foobar",
"version": "1.0.0",
"description": "desc"
}
/* eslint-env mocha */
'use strict'
const { expect } = require('../../../test/test-utils')
const aggregateContent = require('../lib/index')
const FixtureRepo = require('./repo-utils')
const path = require('path')
const _ = require('lodash')
const del = require('del')
function testAll (testFunction, count = 1) {
function test (fixtureRepoOptions) {
const repos = Array.from({ length: count }).map(() => new FixtureRepo(fixtureRepoOptions))
return testFunction(...repos)
}
it('on local repo', () => test({ isRemote: false, isBare: false }))
it('on local bare repo', () => test({ isRemote: false, isBare: true }))
it('on remote repo', () => test({ isRemote: true, isBare: false }))
it('on remote bare repo', () => test({ isRemote: true, isBare: true }))
}
function cleanRepos () {
del.sync('.git-cache')
del.sync(path.resolve(__dirname, 'repos', '*'), { dot: true })
}
describe('aggregateContent()', () => {
let playbook
beforeEach(() => {
cleanRepos()
playbook = { content: { sources: [] } }
})
afterEach(cleanRepos)
// Read & validate docs-component.yml
describe('should throw if docs-component.yml cannot be found', () => {
testAll(async (repo) => {
await repo.initRepo({})
playbook.content.sources.push({ location: repo.location })
const corpus = aggregateContent(playbook)
return expect(corpus).to.be.rejectedWith('docs-component.yml not found')
})
})
describe('should throw if docs-component.yml does not define a name', () => {
testAll(async (repo) => {
await repo.initRepo({ version: 'v1.0.0' })
playbook.content.sources.push({ location: repo.location })
const corpus = aggregateContent(playbook)
return expect(corpus).to.be.rejectedWith('docs-component.yml is missing a name')
})
})
describe('should throw if docs-component.yml does not define a version', () => {
testAll(async (repo) => {
await repo.initRepo({ name: 'the-component' })
playbook.content.sources.push({ location: repo.location })
const corpus = aggregateContent(playbook)
return expect(corpus).to.be.rejectedWith('docs-component.yml is missing a version')
})
})
describe('should read properties from docs-component.yml', () => {
testAll(async (repo) => {
await repo.initRepo({
name: 'the-component',
title: 'The Component',
version: 'v1.2.3',
nav: ['nav-one.adoc', 'nav-two.adoc'],
})
playbook.content.sources.push({ location: repo.location })
const corpus = aggregateContent(playbook)
return expect(corpus)
.to.be.fulfilled()
.then((theCorpus) => {
expect(theCorpus).to.have.lengthOf(1)
expect(theCorpus[0]).to.deep.include({
name: 'the-component',
title: 'The Component',
version: 'v1.2.3',
nav: ['nav-one.adoc', 'nav-two.adoc'],
})
})
})
})
describe('should read properties from docs-component.yml located at specified subpath', () => {
testAll(async (repo) => {
await repo.initRepo({
name: 'the-component',
title: 'The Component',
version: 'v1.2.3',
nav: ['nav-one.adoc', 'nav-two.adoc'],
subpath: 'docs',
})
playbook.content.sources.push({
location: repo.location,
subpath: repo.subpath,
})
const corpus = aggregateContent(playbook)
return expect(corpus)
.to.be.fulfilled()
.then((theCorpus) => {
expect(theCorpus).to.have.lengthOf(1)
expect(theCorpus[0]).to.deep.include({
name: 'the-component',
title: 'The Component',
version: 'v1.2.3',
nav: ['nav-one.adoc', 'nav-two.adoc'],
})
})
})
})
describe('should discover components across multiple repositories', () => {
testAll(async (theComponent, theOtherComponent) => {
await theComponent.initRepo({ name: 'the-component', title: 'The Component', version: 'v1.2.3' })
playbook.content.sources.push({ location: theComponent.location })
await theOtherComponent.initRepo({ name: 'the-other-component', title: 'The Other Component', version: 'v4.5.6' })
playbook.content.sources.push({ location: theOtherComponent.location })
const corpus = aggregateContent(playbook)
return expect(corpus)
.to.be.fulfilled()
.then((theCorpus) => {
expect(theCorpus).to.have.lengthOf(2)
expect(theCorpus[0]).to.deep.include({ name: 'the-component', title: 'The Component', version: 'v1.2.3' })
expect(theCorpus[1]).to.deep.include({
name: 'the-other-component',
title: 'The Other Component',
version: 'v4.5.6',
})
})
}, 2)
})
// Filter branches
async function initRepoWithBranches (repo) {
await repo.initRepo({ name: 'the-component', version: 'unknown' })
await repo.createBranch({ name: 'the-component', version: 'v1.0.0' })
await repo.createBranch({ name: 'the-component', version: 'v2.0.0' })
await repo.createBranch({ name: 'the-component', version: 'v3.0.0' })
}
describe('should read all branches in source repository', () => {
testAll(async (repo) => {
await initRepoWithBranches(repo)
playbook.content.sources.push({ location: repo.location })
const corpus = aggregateContent(playbook)
return expect(corpus)
.to.be.fulfilled()
.then((theCorpus) => {
expect(theCorpus).to.have.lengthOf(4)
expect(theCorpus[0]).to.deep.include({ name: 'the-component', version: 'unknown' })
expect(theCorpus[1]).to.deep.include({ name: 'the-component', version: 'v1.0.0' })
expect(theCorpus[2]).to.deep.include({ name: 'the-component', version: 'v2.0.0' })
expect(theCorpus[3]).to.deep.include({ name: 'the-component', version: 'v3.0.0' })
})
})
})
describe('should filter branches by exact name', () => {
testAll(async (repo) => {
await initRepoWithBranches(repo)
playbook.content.sources.push({
location: repo.location,
branches: 'master',
})
const corpus = aggregateContent(playbook)
return expect(corpus)
.to.be.fulfilled()
.then((theCorpus) => {
expect(theCorpus).to.have.lengthOf(1)
expect(theCorpus[0]).to.deep.include({ name: 'the-component', version: 'unknown' })
})
})
})
describe('should filter branches using wildcard', () => {
testAll(async (repo) => {
await initRepoWithBranches(repo)
playbook.content.sources.push({
location: repo.location,
branches: 'v*',
})
const corpus = aggregateContent(playbook)
return expect(corpus)
.to.be.fulfilled()
.then((theCorpus) => {
expect(theCorpus).to.have.lengthOf(3)
expect(theCorpus[0]).to.deep.include({ name: 'the-component', version: 'v1.0.0' })
expect(theCorpus[1]).to.deep.include({ name: 'the-component', version: 'v2.0.0' })
expect(theCorpus[2]).to.deep.include({ name: 'the-component', version: 'v3.0.0' })
})
})
})
describe('should filter branches using multiple filters', () => {
testAll(async (repo) => {
await initRepoWithBranches(repo)
playbook.content.sources.push({
location: repo.location,
branches: ['master', 'v1.*', 'v3.*'],
})
const corpus = aggregateContent(playbook)
return expect(corpus)
.to.be.fulfilled()
.then((theCorpus) => {
expect(theCorpus).to.have.lengthOf(3)
expect(theCorpus[0]).to.deep.include({ name: 'the-component', version: 'unknown' })
expect(theCorpus[1]).to.deep.include({ name: 'the-component', version: 'v1.0.0' })
expect(theCorpus[2]).to.deep.include({ name: 'the-component', version: 'v3.0.0' })
})
})
})
async function initRepoWithFiles (repo) {
await repo.initRepo({ name: 'the-component', version: 'v1.2.3' })
await repo.addFixtureFiles([
'modules/ROOT/_attributes.adoc',
'modules/ROOT/content/_attributes.adoc',
'modules/ROOT/content/page-one.adoc',
'modules/ROOT/content/page-two.adoc',
'modules/ROOT/content/topic-a/page-three.adoc',
])
}
// Catalog all files
describe('should catalog all files', () => {
testAll(async (repo) => {
await initRepoWithFiles(repo)
playbook.content.sources.push({ location: repo.location })
const corpus = aggregateContent(playbook)
return expect(corpus)
.to.be.fulfilled()
.then((theCorpus) => {
expect(theCorpus).to.have.lengthOf(1)
expect(theCorpus[0]).to.deep.include({
name: 'the-component',
version: 'v1.2.3',
})
expect(theCorpus[0].files).to.have.lengthOf(7)
expect(theCorpus[0].files[0].path).to.equal('README.adoc')
expect(theCorpus[0].files[1].path).to.equal('docs-component.yml')
expect(theCorpus[0].files[2].path).to.equal('modules/ROOT/_attributes.adoc')
expect(theCorpus[0].files[3].path).to.equal('modules/ROOT/content/_attributes.adoc')
expect(theCorpus[0].files[4].path).to.equal('modules/ROOT/content/page-one.adoc')
expect(theCorpus[0].files[5].path).to.equal('modules/ROOT/content/page-two.adoc')
expect(theCorpus[0].files[6].path).to.equal('modules/ROOT/content/topic-a/page-three.adoc')
})
})
})
describe('should populate files with correct contents', () => {
testAll(async (repo) => {
await initRepoWithFiles(repo)
playbook.content.sources.push({ location: repo.location })
const corpus = aggregateContent(playbook)
return expect(corpus)
.to.be.fulfilled()
.then((theCorpus) => {
expect(theCorpus).to.have.lengthOf(1)
expect(theCorpus[0]).to.deep.include({ name: 'the-component', version: 'v1.2.3' })
const pageOne = _.find(theCorpus[0].files, { path: 'modules/ROOT/content/page-one.adoc' })
expect(pageOne.contents.toString()).to.equal(
[
'= Page One',
'ifndef::env-site,env-github[]',
'include::_attributes.adoc[]',
'endif::[]',
':keywords: foo, bar',
'',
'Hey World!',
'',
].join('\n')
)
})
})
})
describe('should catalog all files when component is located at a subpath', () => {
testAll(async (repo) => {
await repo.initRepo({ name: 'the-component', version: 'v1.2.3', subpath: 'docs' })
await repo.addFixtureFiles(['should-be-ignored.adoc'])
await repo.addFixtureFiles(
[
'modules/ROOT/_attributes.adoc',
'modules/ROOT/content/_attributes.adoc',
'modules/ROOT/content/page-one.adoc',
],
'docs'
)
playbook.content.sources.push({ location: repo.location, subpath: repo.subpath })
const corpus =</