Creating a child task can fail and raise a stack trace

In situations where the OS starts to refuse to create new processes, buildstream will print a variety of stack traces, here are some samples from an automated run of buildstream on a host which had no more pids to hand out:

[00:00:00][????????][track:base/ninja.bst            ] BUG     gnome/base-ninja/????????-track.15200.log

    An unhandled exception occured:
    
    Traceback (most recent call last):
      File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 265, in child_action
        result = self.action(element)
      File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/trackqueue.py", line 46, in process
        return element._track()
      File "/usr/local/lib/python3.5/dist-packages/buildstream/element.py", line 1012, in _track
        new_ref = source._track()
      File "/usr/local/lib/python3.5/dist-packages/buildstream/source.py", line 265, in _track
        new_ref = self.track()
      File "/usr/local/lib/python3.5/dist-packages/buildstream/plugins/sources/git.py", line 265, in track
        self.mirror.fetch()
      File "/usr/local/lib/python3.5/dist-packages/buildstream/plugins/sources/git.py", line 114, in fetch
        cwd=self.mirror)
      File "/usr/local/lib/python3.5/dist-packages/buildstream/plugin.py", line 403, in call
        exit_code, _ = self.__call(*popenargs, fail=fail, **kwargs)
      File "/usr/local/lib/python3.5/dist-packages/buildstream/plugin.py", line 514, in __call
        exit_code, output = utils._call(*popenargs, **kwargs)
      File "/usr/local/lib/python3.5/dist-packages/buildstream/utils.py", line 805, in _call
        process = subprocess.Popen(*popenargs, **kwargs)
      File "/usr/lib/python3.5/subprocess.py", line 947, in __init__
        restore_signals, start_new_session)
      File "/usr/lib/python3.5/subprocess.py", line 1490, in _execute_child
        restore_signals, start_new_session, preexec_fn)
    BlockingIOError: [Errno 11] Resource temporarily unavailable

Process Process-5:
Traceback (most recent call last):
  File "/usr/lib/python3.5/multiprocessing/process.py", line 249, in _bootstrap
    self.run()
  File "/usr/lib/python3.5/multiprocessing/process.py", line 93, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 297, in child_action
    logfile=filename)
  File "/usr/lib/python3.5/contextlib.py", line 77, in __exit__
    self.gen.throw(type, value, traceback)
  File "/usr/local/lib/python3.5/dist-packages/buildstream/_signals.py", line 124, in suspendable
    yield
  File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 297, in child_action
    logfile=filename)
  File "/usr/lib/python3.5/contextlib.py", line 77, in __exit__
    self.gen.throw(type, value, traceback)
  File "/usr/local/lib/python3.5/dist-packages/buildstream/element.py", line 1281, in _logging_enabled
    yield self.__log_path
  File "/usr/lib/python3.5/contextlib.py", line 77, in __exit__
    self.gen.throw(type, value, traceback)
  File "/usr/local/lib/python3.5/dist-packages/buildstream/_signals.py", line 64, in terminator
    yield
  File "/usr/local/lib/python3.5/dist-packages/buildstream/element.py", line 1281, in _logging_enabled
    yield self.__log_path
  File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 253, in child_action
    logfile=filename)
  File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 217, in message
    **args))
  File "/usr/local/lib/python3.5/dist-packages/buildstream/context.py", line 271, in _message
    self._message_handler(message, context=self)
  File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 368, in child_message_handler
    self.queue.put(Envelope('message', message))
  File "/usr/lib/python3.5/multiprocessing/queues.py", line 87, in put
    self._start_thread()
  File "/usr/lib/python3.5/multiprocessing/queues.py", line 169, in _start_thread
    self._thread.start()
  File "/usr/lib/python3.5/threading.py", line 844, in start
    _start_new_thread(self._bootstrap, ())
RuntimeError: can't start new thread

Exception in callback Job.parent_recv()
handle: <Handle Job.parent_recv()>
Traceback (most recent call last):
  File "/usr/lib/python3.5/asyncio/events.py", line 125, in _run
    self._callback(*self._args)
  File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 393, in parent_recv
    self.parent_process_queue()
  File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 390, in parent_process_queue
    self.parent_process_envelope(envelope)
  File "/usr/local/lib/python3.5/dist-packages/buildstream/_scheduler/job.py", line 380, in parent_process_envelope
    self.scheduler.context._message(envelope.message)
  File "/usr/local/lib/python3.5/dist-packages/buildstream/context.py", line 271, in _message
    self._message_handler(message, context=self)
  File "/usr/local/lib/python3.5/dist-packages/buildstream/_frontend/main.py", line 902, in message_handler
    text = self.logger.render(message)
  File "/usr/local/lib/python3.5/dist-packages/buildstream/_frontend/widget.py", line 375, in render
    log_content = self.read_last_lines(message.logfile)
  File "/usr/local/lib/python3.5/dist-packages/buildstream/_frontend/widget.py", line 392, in read_last_lines
    output = subprocess.check_output([tail_command, '-n', str(self.log_lines), logfile])
  File "/usr/lib/python3.5/subprocess.py", line 626, in check_output
    **kwargs).stdout
  File "/usr/lib/python3.5/subprocess.py", line 693, in run
    with Popen(*popenargs, **kwargs) as process:
  File "/usr/lib/python3.5/subprocess.py", line 947, in __init__
    restore_signals, start_new_session)
  File "/usr/lib/python3.5/subprocess.py", line 1490, in _execute_child
    restore_signals, start_new_session, preexec_fn)
BlockingIOError: [Errno 11] Resource temporarily unavailable