regression.py 9.65 KB
Newer Older
o9000's avatar
o9000 committed
1 2
#!/usr/bin/env python2

o9000's avatar
o9000 committed
3 4 5 6
from __future__ import print_function

import __builtin__

o9000's avatar
o9000 committed
7 8 9
import sys
reload(sys)
sys.setdefaultencoding('utf8')
o9000's avatar
o9000 committed
10
import argparse
o9000's avatar
o9000 committed
11 12 13 14 15 16 17 18 19 20 21 22
import datetime
import os
import signal
import subprocess
import time


display = "99"
devnull = open(os.devnull, "r+")
ok = ":white_check_mark:"
warning = ":warning:"
error = ":negative_squared_cross_mark:"
o9000's avatar
o9000 committed
23
stress_duration = 10
o9000's avatar
o9000 committed
24
repeats = 1
o9000's avatar
o9000 committed
25 26


o9000's avatar
o9000 committed
27
def print(*args, **kwargs):
o9000's avatar
o9000 committed
28 29 30 31 32 33 34
  if "end" not in kwargs:
    kwargs["end"] = ""
    r = __builtin__.print(*args, **kwargs)
    __builtin__.print("  ")
  else:
    r = __builtin__.print(*args, **kwargs)
    __builtin__.print("\n", end="")
o9000's avatar
o9000 committed
35 36 37
  return r


o9000's avatar
o9000 committed
38 39 40 41
def print_err(*args, **kwargs):
  print(*args, file=sys.stderr, **kwargs)


o9000's avatar
o9000 committed
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
def run(cmd, output=False):
  return subprocess.Popen(cmd,
                          stdin=devnull,
                          stdout=devnull if not output else subprocess.PIPE,
                          stderr=devnull if not output else subprocess.STDOUT,
                          shell=isinstance(cmd, basestring),
                          close_fds=True,
                          preexec_fn=os.setsid)


def stop(p):
  os.killpg(os.getpgid(p.pid), signal.SIGTERM)


def sleep(n):
  while n > 0:
    sys.stderr.write(".")
    sys.stderr.flush()
    time.sleep(1)
    n -= 1


o9000's avatar
o9000 committed
64
def install_deps_ubuntu():
o9000's avatar
o9000 committed
65
  p = run(["sudo", "bash", "-c", "apt-get update; apt-get -y build-dep tint2; apt-get install -y git xvfb xsettingsd openbox compton x11-utils gnome-calculator"])
o9000's avatar
o9000 committed
66 67 68 69 70 71
  out, _ = p.communicate()
  if p.returncode != 0:
    print_err("Process exited with code:", p.returncode, "and output:", out)
    raise RuntimeError("install_deps() failed!")


o9000's avatar
o9000 committed
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111
def start_xvfb():
  stop_xvfb()
  xvfb = run(["Xvfb", ":{0}".format(display), "-screen", "0", "1280x720x24", "-nolisten", "tcp", "-dpi", "96"])
  if xvfb.poll() != None:
    raise RuntimeError("Xvfb failed to start")
  os.environ["DISPLAY"] = ":{0}".format(display)
  return xvfb


def stop_xvfb():
  run("kill $(netstat -ap 2>/dev/null | grep X{0} | grep LISTENING | grep -o '[0-9]*/Xvfb' | head -n 1 | cut -d / -f 1) 1>/dev/null 2>/dev/null ".format(display)).wait()


def start_xsettings():
  return run(["xsettingsd", "-c", "./configs/xsettingsd.conf"])


def start_wm():
  return run(["openbox", "--replace", "--config-file", "./configs/openbox.xml"])

def start_compositor():
  return run(["compton", "--config", "./configs/compton.conf"])


def start_stressors():
  stressors = []
  stressors.append(run(["./workspaces-stress.sh"]))
  return stressors


def stop_stressors(stressors):
  for s in stressors:
    stop(s)


def compute_min_med_fps(out):
  samples = []
  for line in out.split("\n"):
    if "fps = " in line:
      fps = float(line.split("fps = ", 1)[-1].split(" ")[0])
o9000's avatar
o9000 committed
112 113
      if fps > 0:
        samples.append(fps)
o9000's avatar
o9000 committed
114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
  samples.sort()
  return min(samples), samples[len(samples)/2]


def get_mem_usage(pid):
  value = None
  with open("/proc/{0}/status".format(pid)) as f:
    for line in f:
      if line.startswith("VmRSS:"):
        rss = line.split(":", 1)[-1].strip()
        value, multiplier = rss.split(" ")
        value = float(value)
        if multiplier == "kB":
          value *= 1024
        else:
          raise RuntimeError("Could not parse /proc/[pid]/status")
  if not value:
    raise RuntimeError("Could not parse /proc/[pid]/status")
  return value * 1.0e-6


def find_asan_leaks(out):
  traces = []
  trace = None
  for line in out.split("\n"):
    line = line.strip()
    if " leak of " in line and " allocated from:" in line:
      trace = []
    if trace != None:
      if line.startswith("#"):
        trace.append(line)
      else:
        if any([ "tint2" in frame for frame in trace ]):
          traces.append(trace)
        trace = None
  return traces


def test(tint2path, config):
  start_xvfb()
  sleep(1)
  start_xsettings()
  start_wm()
  sleep(1)
  os.environ["DEBUG_FPS"] = "1"
  os.environ["ASAN_OPTIONS"] = "detect_leaks=1"
o9000's avatar
o9000 committed
160
  tint2 = run([tint2path, "-c", config], True)
o9000's avatar
o9000 committed
161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
  if tint2.poll() != None:
    raise RuntimeError("tint2 failed to start")
  sleep(1)
  # Handle late compositor start
  compton = start_compositor()
  sleep(2)
  # Stress test with compositor on
  stressors = start_stressors()
  sleep(stress_duration)
  stop_stressors(stressors)
  # Handle compositor stopping
  stop(compton)
  # Stress test with compositor off
  stressors = start_stressors()
  sleep(stress_duration)
  stop_stressors(stressors)
  # Handle WM restart
  start_wm()
  # Stress test with new WM
  stressors = start_stressors()
  sleep(stress_duration)
  stop_stressors(stressors)
  # Collect info
  mem = get_mem_usage(tint2.pid)
  stop(tint2)
  out, _ = tint2.communicate()
  exitcode = tint2.returncode
  if exitcode != 0:
o9000's avatar
o9000 committed
189 190
    print("tint2 crashed with exit code {0}!".format(exitcode))
    print("Output:")
o9000's avatar
o9000 committed
191
    print("```\n" + out.strip() + "\n```")
o9000's avatar
o9000 committed
192 193 194 195 196
    return
  min_fps, med_fps = compute_min_med_fps(out)
  leaks = find_asan_leaks(out)
  sys.stderr.write("\n")
  mem_status = ok if mem < 20 else warning if mem < 40 else error
o9000's avatar
o9000 committed
197
  print("Memory usage: %.1f %s %s" % (mem, "MB", mem_status))
o9000's avatar
o9000 committed
198
  leak_status = ok if not leaks else error
o9000's avatar
o9000 committed
199
  print("Memory leak count:", len(leaks), leak_status)
o9000's avatar
o9000 committed
200
  for leak in leaks:
o9000's avatar
o9000 committed
201
    print("Memory leak:")
o9000's avatar
o9000 committed
202
    for line in leak:
o9000's avatar
o9000 committed
203
      print(line)
o9000's avatar
o9000 committed
204
  fps_status = ok if min_fps > 60 else warning if min_fps > 40 else error
o9000's avatar
o9000 committed
205
  print("FPS:", "min:", min_fps, "median:", med_fps, fps_status)
o9000's avatar
o9000 committed
206
  if mem_status != ok or leak_status != ok or fps_status != ok:
o9000's avatar
o9000 committed
207
    print("Output:")
o9000's avatar
o9000 committed
208
    print("```\n" + out.strip() + "\n```")
o9000's avatar
o9000 committed
209 210 211
  stop_xvfb()


o9000's avatar
o9000 committed
212
def show_timestamp():
o9000's avatar
o9000 committed
213
  utc_datetime = datetime.datetime.utcnow()
o9000's avatar
o9000 committed
214
  print("Last updated:", utc_datetime.strftime("%Y-%m-%d %H:%M UTC"))
o9000's avatar
o9000 committed
215 216 217


def show_git_info(src_dir):
o9000's avatar
o9000 committed
218
  out, _ = run("cd {0}; git show -s '--format=[%ci] %h %s %d'".format(src_dir), True).communicate()
o9000's avatar
o9000 committed
219
  print("Last commit:", out.strip())
o9000's avatar
o9000 committed
220
  diff, _ = run("cd {0}; git diff".format(src_dir), True).communicate()
o9000's avatar
o9000 committed
221
  diff = diff.strip()
o9000's avatar
o9000 committed
222
  diff_staged, _ = run("cd {0}; git diff --staged".format(src_dir), True).communicate()
o9000's avatar
o9000 committed
223 224
  diff_staged = diff_staged.strip()
  if diff or diff_staged:
o9000's avatar
o9000 committed
225
    print("Repository not clean", warning)
o9000's avatar
o9000 committed
226
    if diff:
o9000's avatar
o9000 committed
227
      print("Diff:")
o9000's avatar
o9000 committed
228
      print("```\n" + diff + "\n```")
o9000's avatar
o9000 committed
229
    if diff_staged:
o9000's avatar
o9000 committed
230
      print("Diff staged:")
o9000's avatar
o9000 committed
231
      print("```\n" + diff_staged + "\n```")
o9000's avatar
o9000 committed
232 233 234


def show_system_info():
o9000's avatar
o9000 committed
235
  out, _ = run("lsb_release -sd", True).communicate()
o9000's avatar
o9000 committed
236
  out = out.strip()
o9000's avatar
o9000 committed
237
  print("System:", out)
o9000's avatar
o9000 committed
238
  out, _ = run("cat /proc/cpuinfo | grep 'model name' | head -n1 | cut -d ':' -f2", True).communicate()
o9000's avatar
o9000 committed
239
  out = out.strip()
o9000's avatar
o9000 committed
240
  print("Hardware:", out)
o9000's avatar
o9000 committed
241
  out, _ = run("cc --version | head -n1", True).communicate()
o9000's avatar
o9000 committed
242
  out = out.strip()
o9000's avatar
o9000 committed
243
  print("Compiler:", out)
o9000's avatar
o9000 committed
244 245 246


def compile_and_report(src_dir):
o9000's avatar
o9000 committed
247
  print("# Compilation")
o9000's avatar
o9000 committed
248
  cmake_flags = "-DCMAKE_BUILD_TYPE=Debug -DENABLE_ASAN=ON '-DCMAKE_CXX_FLAGS_DEBUG=-O0 -g3 -gdwarf-2 -fsanitize=address -fno-common -fno-omit-frame-pointer -rdynamic -Wshadow' '-DCMAKE_EXE_LINKER_FLAGS=-O0 -g3 -gdwarf-2 -fsanitize=address -fno-common -fno-omit-frame-pointer -rdynamic -fuse-ld=gold'"
o9000's avatar
o9000 committed
249
  print("Flags:", cmake_flags)
o9000's avatar
o9000 committed
250
  start = time.time()
o9000's avatar
o9000 committed
251
  c = run("rm -rf build; mkdir build; cd build; cmake {0} {1} ; make -j7".format(cmake_flags, src_dir), True)
o9000's avatar
o9000 committed
252 253 254
  out, _ = c.communicate()
  duration = time.time() - start
  if c.returncode != 0:
o9000's avatar
o9000 committed
255 256
    print("Status: Failed!", error)
    print("Output:")
o9000's avatar
o9000 committed
257
    print("```\n" + out.strip() + "\n```")
o9000's avatar
o9000 committed
258
    raise RuntimeError("compilation failed")
o9000's avatar
o9000 committed
259
  if "warning:" in out:
o9000's avatar
o9000 committed
260 261
    print("Status: Succeeded with warnings!", warning)
    print("Warnings:")
o9000's avatar
o9000 committed
262
    print("```", end="")
o9000's avatar
o9000 committed
263 264
    for line in out.split("\n"):
      if "warning:" in line:
o9000's avatar
o9000 committed
265 266
        print(line, end="")
    print("```", end="")
o9000's avatar
o9000 committed
267
  else:
o9000's avatar
o9000 committed
268
    print("Status: Succeeded in %.1f seconds" % (duration,), ok)
o9000's avatar
o9000 committed
269 270 271 272 273


def run_test(config, index):
  print("# Test", index)
  print("Config: [{0}]({1})".format(config.split("/")[-1].replace(".tint2rc", ""), "https://gitlab.com/o9000/tint2/blob/master/test/" + config))
o9000's avatar
o9000 committed
274 275
  for i in range(repeats):
    test("./build/tint2", config)
o9000's avatar
o9000 committed
276 277 278


def run_tests():
o9000's avatar
o9000 committed
279 280 281 282 283 284
  configs = []
  configs += ["./configs/tint2/" +s for s in os.listdir("./configs/tint2") ]
  configs += ["../themes/" + s for s in os.listdir("../themes")]
  index = 0
  for config in configs:
    index += 1
o9000's avatar
o9000 committed
285
    run_test(config, index)
o9000's avatar
o9000 committed
286
    print("")
o9000's avatar
o9000 committed
287 288


o9000's avatar
o9000 committed
289 290 291 292
def get_default_src_dir():
  return os.path.realpath(os.path.dirname(os.path.realpath(__file__)) + "/../")


o9000's avatar
o9000 committed
293
def check_busy():
o9000's avatar
o9000 committed
294 295
  out, _ = run("top -bn5 | grep 'Cpu(s)' | grep -o '[0-9\.]* id' | cut -d ' ' -f 1", True).communicate()
  load_samples = []
o9000's avatar
o9000 committed
296
  for line in out.split("\n"):
o9000's avatar
o9000 committed
297 298 299 300 301
    line = line.strip()
    if line:
      load_samples.append(100. - float(line))
  load_samples.sort()
  load = load_samples[len(load_samples)/2]
o9000's avatar
o9000 committed
302
  if load > 10.0:
o9000's avatar
o9000 committed
303
    raise RuntimeError("The system appears busy. Load: %f.1%%." % (load,))
o9000's avatar
o9000 committed
304 305


o9000's avatar
o9000 committed
306 307 308 309
def checkout(version):
  p = run("rm -rf tmpclone; git clone https://gitlab.com/o9000/tint2.git tmpclone; cd tmpclone; git checkout {0}".format(version), True)
  out, _ = p.communicate()
  if p.returncode != 0:
o9000's avatar
o9000 committed
310
    print_err(out)
o9000's avatar
o9000 committed
311 312 313
    raise RuntimeError("git clone failed!")


o9000's avatar
o9000 committed
314 315
def main():
  parser = argparse.ArgumentParser()
o9000's avatar
o9000 committed
316
  parser.add_argument("--src_dir", default=get_default_src_dir())
o9000's avatar
o9000 committed
317
  parser.add_argument("--for_version", default="HEAD")
o9000's avatar
o9000 committed
318 319
  parser.add_argument("--install_deps", dest="install_deps", action="store_true")
  parser.set_defaults(install_deps=False)
o9000's avatar
o9000 committed
320
  args = parser.parse_args()
o9000's avatar
o9000 committed
321 322
  if args.install_deps:
    install_deps_ubuntu()
o9000's avatar
o9000 committed
323 324 325 326
  if args.for_version != "HEAD":
    checkout(args.for_version)
    args.src_dir = "./tmpclone"
  args.src_dir = os.path.realpath(args.src_dir)
o9000's avatar
o9000 committed
327
  stop_xvfb()
o9000's avatar
o9000 committed
328
  check_busy()
o9000's avatar
o9000 committed
329 330 331 332 333 334
  show_timestamp()
  show_git_info(args.src_dir)
  show_system_info()
  compile_and_report(args.src_dir)
  run_tests()

o9000's avatar
o9000 committed
335 336

if __name__ == "__main__":
o9000's avatar
o9000 committed
337
  sys.stdout = os.fdopen(sys.stdout.fileno(), "w", 0)
o9000's avatar
o9000 committed
338
  main()