Creating a child task can fail and raise a stack trace
In situations where the OS starts to refuse to create new processes, buildstream will print a variety of stack traces, here are some samples from an automated run of buildstream on a host which had no more pids to hand out:
[00:00:00][????????][track:base/ninja.bst ] BUG gnome/base-ninja/????????-track.15200.log
An unhandled exception occured:
Traceback (most recent call last):
File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 265, in child_action
result = self.action(element)
File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/trackqueue.py", line 46, in process
return element._track()
File "/usr/local/lib/python3.5/dist-packages/buildstream/element.py", line 1012, in _track
new_ref = source._track()
File "/usr/local/lib/python3.5/dist-packages/buildstream/source.py", line 265, in _track
new_ref = self.track()
File "/usr/local/lib/python3.5/dist-packages/buildstream/plugins/sources/git.py", line 265, in track
self.mirror.fetch()
File "/usr/local/lib/python3.5/dist-packages/buildstream/plugins/sources/git.py", line 114, in fetch
cwd=self.mirror)
File "/usr/local/lib/python3.5/dist-packages/buildstream/plugin.py", line 403, in call
exit_code, _ = self.__call(*popenargs, fail=fail, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/buildstream/plugin.py", line 514, in __call
exit_code, output = utils._call(*popenargs, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/buildstream/utils.py", line 805, in _call
process = subprocess.Popen(*popenargs, **kwargs)
File "/usr/lib/python3.5/subprocess.py", line 947, in __init__
restore_signals, start_new_session)
File "/usr/lib/python3.5/subprocess.py", line 1490, in _execute_child
restore_signals, start_new_session, preexec_fn)
BlockingIOError: [Errno 11] Resource temporarily unavailable
Process Process-5:
Traceback (most recent call last):
File "/usr/lib/python3.5/multiprocessing/process.py", line 249, in _bootstrap
self.run()
File "/usr/lib/python3.5/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 297, in child_action
logfile=filename)
File "/usr/lib/python3.5/contextlib.py", line 77, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/local/lib/python3.5/dist-packages/buildstream/_signals.py", line 124, in suspendable
yield
File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 297, in child_action
logfile=filename)
File "/usr/lib/python3.5/contextlib.py", line 77, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/local/lib/python3.5/dist-packages/buildstream/element.py", line 1281, in _logging_enabled
yield self.__log_path
File "/usr/lib/python3.5/contextlib.py", line 77, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/local/lib/python3.5/dist-packages/buildstream/_signals.py", line 64, in terminator
yield
File "/usr/local/lib/python3.5/dist-packages/buildstream/element.py", line 1281, in _logging_enabled
yield self.__log_path
File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 253, in child_action
logfile=filename)
File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 217, in message
**args))
File "/usr/local/lib/python3.5/dist-packages/buildstream/context.py", line 271, in _message
self._message_handler(message, context=self)
File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 368, in child_message_handler
self.queue.put(Envelope('message', message))
File "/usr/lib/python3.5/multiprocessing/queues.py", line 87, in put
self._start_thread()
File "/usr/lib/python3.5/multiprocessing/queues.py", line 169, in _start_thread
self._thread.start()
File "/usr/lib/python3.5/threading.py", line 844, in start
_start_new_thread(self._bootstrap, ())
RuntimeError: can't start new thread
Exception in callback Job.parent_recv()
handle: <Handle Job.parent_recv()>
Traceback (most recent call last):
File "/usr/lib/python3.5/asyncio/events.py", line 125, in _run
self._callback(*self._args)
File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 393, in parent_recv
self.parent_process_queue()
File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 390, in parent_process_queue
self.parent_process_envelope(envelope)
File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 380, in parent_process_envelope
self.scheduler.context._message(envelope.message)
File "/usr/local/lib/python3.5/dist-packages/buildstream/context.py", line 271, in _message
self._message_handler(message, context=self)
File "/usr/local/lib/python3.5/dist-packages/buildstream/_frontend/main.py", line 902, in message_handler
text = self.logger.render(message)
File "/usr/local/lib/python3.5/dist-packages/buildstream/_frontend/widget.py", line 375, in render
log_content = self.read_last_lines(message.logfile)
File "/usr/local/lib/python3.5/dist-packages/buildstream/_frontend/widget.py", line 392, in read_last_lines
output = subprocess.check_output([tail_command, '-n', str(self.log_lines), logfile])
File "/usr/lib/python3.5/subprocess.py", line 626, in check_output
**kwargs).stdout
File "/usr/lib/python3.5/subprocess.py", line 693, in run
with Popen(*popenargs, **kwargs) as process:
File "/usr/lib/python3.5/subprocess.py", line 947, in __init__
restore_signals, start_new_session)
File "/usr/lib/python3.5/subprocess.py", line 1490, in _execute_child
restore_signals, start_new_session, preexec_fn)
BlockingIOError: [Errno 11] Resource temporarily unavailable