gitlab-ci: Add gfxreconstruct traces support

Signed-off-by: Andres Gomez <agomez@igalia.com>
Reviewed-by: Samuel Pitoiset <samuel.pitoiset@gmail.com>
Reviewed-by: Alexandros Frantzis <alexandros.frantzis@collabora.com>

.gitlab-ci.yml

@@ -131,7 +131,7 @@ x86_test-gl:
 x86_test-vk:
   extends: x86_build
   variables:
-    DEBIAN_TAG: &x86_test-vk "2020-02-03"
+    DEBIAN_TAG: &x86_test-vk "2020-03-04"
   # Can only be triggered manually on personal branches because RADV is the only
   # driver that does Vulkan testing at the moment.
   rules:

.gitlab-ci/container/x86_test-vk.sh

@@ -31,6 +31,7 @@ apt-get install -y --no-remove \
       cmake \
       g++ \
       git \
+      git-lfs \
       gcc \
       libexpat1 \
       libgbm-dev \
@@ -60,6 +61,8 @@ apt-get install -y --no-remove \
       patch \
       pkg-config \
       python3-distutils \
+      python3-pil \
+      python3-yaml \
       python \
       xauth \
       xvfb
@@ -87,7 +90,6 @@ apt-get purge -y \
       cmake \
       g++ \
       gcc \
-      git \
       gnupg \
       libgbm-dev \
       libgles2-mesa-dev \
@@ -101,7 +103,6 @@ apt-get purge -y \
       libxrender-dev \
       meson \
       patch \
-      pkg-config \
-      python
+      pkg-config
 
 apt-get autoremove -y --purge

.gitlab-ci/tracie/dump_trace_images.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 
 # Copyright (c) 2019 Collabora Ltd
+# Copyright © 2019 Valve Corporation.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -35,8 +36,8 @@ def log(severity, msg, end='\n'):
 def log_result(msg):
     print(msg, flush=True)
 
-def run_logged_command(cmd, log_path):
-    ret = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+def run_logged_command(cmd, env, log_path):
+    ret = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env)
     logoutput = ("[dump_trace_images] Running: %s\n" % " ".join(cmd)).encode() + \
                 ret.stdout
     log_path.parent.mkdir(parents=True, exist_ok=True)
@@ -56,6 +57,20 @@ def get_last_apitrace_frame_call(trace_path):
             return int(s[0])
     return -1
 
+def get_last_gfxreconstruct_frame_call(trace_path):
+    # FIXME: It would be great to have another way to get the amount of
+    # traces which wouldn't imply replaying the whole trace:
+    # https://github.com/LunarG/gfxreconstruct/issues/329
+    cmd = ["gfxrecon-replay", str(trace_path)]
+    ret = subprocess.run(cmd, stdout=subprocess.PIPE)
+    for l in reversed(ret.stdout.decode(errors='replace').splitlines()):
+        s = l.split(", ", 2)
+        if len(s) >= 3:
+            c = s[2].split(None, 1)
+            if len(c) >= 1 and c[0].isnumeric():
+                return int(c[0])
+    return -1
+
 def dump_with_apitrace(trace_path, calls, device_name):
     outputdir = str(trace_path.parent / "test" / device_name)
     os.makedirs(outputdir, exist_ok=True)
@@ -65,7 +80,7 @@ def dump_with_apitrace(trace_path, calls, device_name):
     cmd = ["apitrace", "dump-images", "--calls=" + ','.join(calls),
            "-o", outputprefix, str(trace_path)]
     log_path = Path(outputdir) / (trace_path.name + ".log")
-    run_logged_command(cmd, log_path)
+    run_logged_command(cmd, None, log_path)
 
 def dump_with_renderdoc(trace_path, calls, device_name):
     outputdir = str(trace_path.parent / "test" / device_name)
@@ -73,7 +88,33 @@ def dump_with_renderdoc(trace_path, calls, device_name):
     cmd = [str(script_path / "renderdoc_dump_images.py"), str(trace_path), outputdir]
     cmd.extend(calls)
     log_path = Path(outputdir) / (trace_path.name + ".log")
-    run_logged_command(cmd, log_path)
+    run_logged_command(cmd, None, log_path)
+
+def dump_with_gfxreconstruct(trace_path, calls, device_name):
+    from PIL import Image
+    outputdir_path = trace_path.parent / "test" / device_name
+    outputdir_path.mkdir(parents=True, exist_ok=True)
+    outputprefix = str(outputdir_path / trace_path.name) + "-"
+    if len(calls) == 0:
+        # FIXME: The VK_LAYER_LUNARG_screenshot numbers the calls from
+        # 0 to (total-num-calls - 1) while gfxreconstruct does it from
+        # 1 to total-num-calls:
+        # https://github.com/LunarG/gfxreconstruct/issues/284
+        calls = [str(get_last_gfxreconstruct_frame_call(trace_path) - 1)]
+    cmd = ["gfxrecon-replay", str(trace_path)]
+    log_path = outputdir_path / (trace_path.name + ".log")
+    env = os.environ.copy()
+    env["VK_INSTANCE_LAYERS"] = "VK_LAYER_LUNARG_screenshot"
+    env["VK_SCREENSHOT_FRAMES"] = ",".join(calls)
+    env["VK_SCREENSHOT_DIR"] = str(outputdir_path)
+    run_logged_command(cmd, env, log_path)
+    for c in calls:
+        ppm = str(outputdir_path / c) + ".ppm"
+        outputfile = outputprefix + c + ".png"
+        with log_path.open(mode='w') as log:
+            log.write("Writing: %s to %s" % (ppm, outputfile))
+        Image.open(ppm).save(outputfile)
+        os.remove(ppm)
 
 def dump_with_testtrace(trace_path, calls, device_name):
     from PIL import Image
@@ -99,6 +140,8 @@ def dump_from_trace(trace_path, calls, device_name):
             dump_with_apitrace(trace_path, calls, device_name)
         elif trace_type == TraceType.RENDERDOC:
             dump_with_renderdoc(trace_path, calls, device_name)
+        elif trace_type == TraceType.GFXRECONSTRUCT:
+            dump_with_gfxreconstruct(trace_path, calls, device_name)
         elif trace_type == TraceType.TESTTRACE:
             dump_with_testtrace(trace_path, calls, device_name)
         else:

.gitlab-ci/tracie/traceutil.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2019 Collabora Ltd
+# Copyright © 2019 Valve Corporation.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -28,11 +29,13 @@ class TraceType(Enum):
     UNKNOWN = auto()
     APITRACE = auto()
     RENDERDOC = auto()
+    GFXRECONSTRUCT = auto()
     TESTTRACE = auto()
 
 _trace_type_info_map = {
     TraceType.APITRACE : ("apitrace", ".trace"),
     TraceType.RENDERDOC : ("renderdoc", ".rdc"),
+    TraceType.GFXRECONSTRUCT : ("gfxreconstruct", ".gfxr"),
     TraceType.TESTTRACE : ("testtrace", ".testtrace")
 }