Commit 8b902209 authored by locallycompact-ct's avatar locallycompact-ct

Build logic reduction for assemblages

parent 1b37a3a7
......@@ -22,12 +22,13 @@ import sys
import fcntl
from ybd import app, cache, sandbox
from ybd.app import cleanup, config, log, RetryException, setup, spawn, timer
from ybd.assembly import compose
from ybd.assembly import compose, build
from ybd.deployment import deploy
from ybd.pots import Pots
from ybd.concourse import Pipeline
from ybd.release_note import do_release_note
from defslib import MorphologyResolver, Actuator
from ybd.defaults import Defaults
import sandboxlib
import yaml
......@@ -60,6 +61,8 @@ with timer('TOTAL'):
log('TARGET', 'Target is %s' % config['target'], config['arch'])
with timer('DEFINITIONS', 'parsing %s' % config['def-version']):
m = MorphologyResolver(config['defdir'])
app.defs = m
app.defaults = Defaults()
target = m.lookup(config['target'])
if config.get('mode', 'normal') == 'parse-only':
......@@ -73,6 +76,7 @@ with timer('TOTAL'):
defaults=m.defaults)
with timer('CACHE-KEYS', 'cache-key calculations'):
x.cache_enrich_assemblage(target)
x.flatten_assemblage(target)
if 'release-note' in config:
do_release_note(config['release-note'])
......@@ -94,18 +98,20 @@ with timer('TOTAL'):
while True:
try:
compose(target)
for y in target['contents']:
with sandbox.setup(y):
build(x,y,target['contents'])
break
except KeyboardInterrupt:
log(target, 'Interrupted by user')
os._exit(1)
except RetryException:
pass
except:
except Exception as e:
import traceback
traceback.print_exc()
log(target, 'Exiting: uncaught exception')
os._exit(1)
raise e
if config.get('reproduce'):
log('REPRODUCED',
......
......@@ -31,10 +31,6 @@ from ybd.splitting import write_metadata, install_split_artifacts
def compose(dn):
'''Work through defs tree, building and assembling until target exists'''
if type(dn) is not dict:
dn = app.defs.get(dn)
# if we can't calculate cache key, we can't create this component
if cache_key(dn) is False:
if 'tried' not in dn:
log(dn, 'No cache_key, so skipping compose')
......@@ -83,8 +79,7 @@ def install_contents(dn, contents=None):
log(dn, 'Installing contents\n', contents, verbose=True)
shuffle(contents)
for it in contents:
item = app.defs.get(it)
for item in contents:
if os.path.exists(os.path.join(dn['sandbox'],
'baserock', item['name'] + '.meta')):
# content has already been installed
......@@ -107,32 +102,20 @@ def install_contents(dn, contents=None):
def install_dependencies(dn, dependencies=None):
'''Install recursed dependencies of dn into dn's sandbox.'''
if dependencies is None:
dependencies = dn.get('build-depends', [])
log(dn, 'Installing dependencies\n', dependencies, verbose=True)
shuffle(dependencies)
for it in dependencies:
dependency = app.defs.get(it)
for dependency in dependencies:
if os.path.exists(os.path.join(dn['sandbox'], 'baserock',
dependency['name'] + '.meta')):
# dependency has already been installed
log(dn, 'Already did', dependency['name'], verbose=True)
continue
install_dependencies(dn, dependency.get('build-depends', []))
if (it in dn['build-depends']) or \
(dependency.get('build-mode', 'staging') ==
dn.get('build-mode', 'staging')):
compose(dependency)
if dependency.get('contents'):
install_dependencies(dn, dependency['contents'])
sandbox.install(dn, dependency)
sandbox.install(dn, dependency)
if config.get('log-verbose'):
sandbox.list_files(dn)
def build(dn):
def build(actuator, dn, contents):
'''Create an artifact for a single component and add it to the cache'''
if get_cache(dn):
......@@ -140,9 +123,10 @@ def build(dn):
with claim(dn):
if dn.get('kind', 'chunk') == 'chunk':
install_dependencies(dn)
s = actuator.lens(dn, contents)
install_dependencies(dn, s['supports'])
with timer(dn, 'build of %s' % dn['cache']):
run_build(dn)
run_build(dn, s['supports'])
with timer(dn, 'artifact creation'):
......@@ -153,7 +137,7 @@ def build(dn):
cache(dn)
def run_build(dn):
def run_build(dn, supports):
''' This is where we run ./configure, make, make install (for example).
By the time we get here, all dependencies for component have already
been assembled.
......@@ -170,11 +154,11 @@ def run_build(dn):
repos.checkout(dn)
dn['SOURCE_DATE_EPOCH'] = repos.source_date_epoch(dn['checkout'])
get_build_commands(dn)
env_vars = sandbox.env_vars_for_build(dn)
#get_build_commands(dn)
env_vars = sandbox.env_vars_for_build(dn, supports)
log(dn, 'Logging build commands to %s' % dn['log'])
for build_step in app.defs.defaults.build_steps:
for build_step in app.defaults.build_steps:
if dn.get(build_step):
log(dn, 'Running', build_step)
for command in dn.get(build_step, []):
......@@ -242,18 +226,6 @@ def get_build_commands(dn):
return
exit = True if config.get('check-definitions') == 'exit' else False
if 'build-system' in dn:
bs = dn['build-system']
log(dn, 'Defined build system is', bs)
else:
files = os.listdir(dn['checkout'])
bs = app.defs.defaults.detect_build_system(files)
if bs == 'manual' and 'install-commands' not in dn:
if dn.get('kind', 'chunk') == 'chunk':
print(dn)
log(dn, 'WARNING: No install-commands, manual build-system',
exit=exit)
log(dn, 'WARNING: Assumed build system is', bs)
for build_step in app.defs.defaults.build_steps:
if dn.get(build_step, None) is None:
......
......@@ -57,11 +57,12 @@ def setup(dn):
yield
except app.RetryException as e:
raise e
except:
except Exception as e:
import traceback
app.log(dn, 'ERROR: surprise exception in sandbox', '')
traceback.print_exc()
app.log(dn, 'Sandbox debris is at', dn['sandbox'], exit=True)
app.log(dn, 'Sandbox debris is at', dn['sandbox'])
raise e
finally:
pass
......@@ -259,7 +260,7 @@ def ccache_mounts(dn, ccache_target):
return mounts
def env_vars_for_build(dn):
def env_vars_for_build(dn, dependencies):
env = {}
extra_path = []
......@@ -277,9 +278,8 @@ def env_vars_for_build(dn):
prefixes = []
for name in dn.get('build-depends', []):
dependency = app.defs.get(name)
prefixes.append(dependency.get('prefix', '/usr'))
for it in dependencies:
prefixes.append(it.get('prefix', '/usr'))
prefixes = set(prefixes)
for prefix in prefixes:
if prefix:
......@@ -306,7 +306,7 @@ def env_vars_for_build(dn):
env['TZ'] = 'UTC'
arch = app.config['arch']
cpu = app.config['cpu']
cpu = app.defaults.cpus.get(arch, arch)
abi = ''
if arch.startswith(('armv7', 'armv5')):
abi = 'eabi'
......
......@@ -165,7 +165,7 @@ def compile_rules(dn):
regexps = []
splits = {}
split_rules = dn.get('products', [])
default_rules = app.defs.defaults.get_split_rules(dn.get('kind', 'chunk'))
default_rules = app.defaults.get_split_rules(dn.get('kind', 'chunk'))
for rules in split_rules, default_rules:
for rule in rules:
regexp = re.compile('^(?:' + '|'.join(rule.get('include')) + ')$')
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment