diff --git a/src/mesa/drivers/dri/i965/brw_context.h b/src/mesa/drivers/dri/i965/brw_context.h index 0aa27995ab9..c67dc7ceafb 100644 --- a/src/mesa/drivers/dri/i965/brw_context.h +++ b/src/mesa/drivers/dri/i965/brw_context.h @@ -1114,7 +1114,59 @@ struct brw_context struct brw_perf_query_info *queries; int n_queries; + /* The i915 perf stream we open to setup + enable the OA counters */ + int oa_stream_fd; + + /* An i915 perf stream fd gives exclusive access to the OA unit that will + * report counter snapshots for a specific counter set/profile in a + * specific layout/format so we can only start OA queries that are + * compatible with the currently open fd... + */ + int current_oa_metrics_set_id; + int current_oa_format; + + /* List of buffers containing OA reports */ + struct exec_list sample_buffers; + + /* Cached list of empty sample buffers */ + struct exec_list free_sample_buffers; + + int n_active_oa_queries; int n_active_pipeline_stats_queries; + + /* The number of queries depending on running OA counters which + * extends beyond brw_end_perf_query() since we need to wait until + * the last MI_RPC command has parsed by the GPU. + * + * Accurate accounting is important here as emitting an + * MI_REPORT_PERF_COUNT command while the OA unit is disabled will + * effectively hang the gpu. + */ + int n_oa_users; + + /* To help catch an spurious problem with the hardware or perf + * forwarding samples, we emit each MI_REPORT_PERF_COUNT command + * with a unique ID that we can explicitly check for... + */ + int next_query_start_report_id; + + /** + * An array of queries whose results haven't yet been assembled + * based on the data in buffer objects. + * + * These may be active, or have already ended. However, the + * results have not been requested. + */ + struct brw_perf_query_object **unaccumulated; + int unaccumulated_elements; + int unaccumulated_array_size; + + /* The total number of query objects so we can relinquish + * our exclusive access to perf if the application deletes + * all of its objects. (NB: We only disable perf while + * there are no active queries) + */ + int n_query_instances; } perfquery; int num_atoms[BRW_NUM_PIPELINES]; diff --git a/src/mesa/drivers/dri/i965/brw_performance_query.c b/src/mesa/drivers/dri/i965/brw_performance_query.c index 6cd5bf4caa5..64b3beeeae0 100644 --- a/src/mesa/drivers/dri/i965/brw_performance_query.c +++ b/src/mesa/drivers/dri/i965/brw_performance_query.c @@ -26,12 +26,21 @@ * * Implementation of the GL_INTEL_performance_query extension. * - * Currently this driver only exposes the 64bit Pipeline Statistics - * Registers for Gen6+, with support for Observability Counters to be - * added later for Gen7.5+ + * Currently there are two possible counter sources exposed here: + * + * On Gen6+ hardware we have numerous 64bit Pipeline Statistics Registers + * that we can snapshot at the beginning and end of a query. + * + * On Gen7.5+ we have Observability Architecture counters which are + * covered in separate document from the rest of the PRMs. It is available at: + * https://01.org/linuxgraphics/documentation/driver-documentation-prms + * => 2013 Intel Core Processor Family => Observability Performance Counters + * (This one volume covers Sandybridge, Ivybridge, Baytrail, and Haswell, + * though notably we currently only support OA counters for Haswell+) */ #include +#include #include #include @@ -40,6 +49,9 @@ #include #include +#include +#include + #include "main/hash.h" #include "main/macros.h" #include "main/mtypes.h" @@ -47,14 +59,145 @@ #include "util/bitset.h" #include "util/ralloc.h" +#include "util/hash_table.h" +#include "util/list.h" #include "brw_context.h" #include "brw_defines.h" #include "brw_performance_query.h" +#include "brw_oa_hsw.h" #include "intel_batchbuffer.h" #define FILE_DEBUG_FLAG DEBUG_PERFMON +/* + * The largest OA format we can use on Haswell includes: + * 1 timestamp, 45 A counters, 8 B counters and 8 C counters. + */ +#define MAX_OA_REPORT_COUNTERS 62 + +#define I915_PERF_OA_SAMPLE_SIZE (8 + /* drm_i915_perf_record_header */ \ + 256) /* OA counter report */ + +/** + * Periodic OA samples are read() into these buffer structures via the + * i915 perf kernel interface and appended to the + * brw->perfquery.sample_buffers linked list. When we process the + * results of an OA metrics query we need to consider all the periodic + * samples between the Begin and End MI_REPORT_PERF_COUNT command + * markers. + * + * 'Periodic' is a simplification as there are other automatic reports + * written by the hardware also buffered here. + * + * Considering three queries, A, B and C: + * + * Time ----> + * ________________A_________________ + * | | + * | ________B_________ _____C___________ + * | | | | | | + * + * And an illustration of sample buffers read over this time frame: + * [HEAD ][ ][ ][ ][ ][ ][ ][ ][TAIL ] + * + * These nodes may hold samples for query A: + * [ ][ ][ A ][ A ][ A ][ A ][ A ][ ][ ] + * + * These nodes may hold samples for query B: + * [ ][ ][ B ][ B ][ B ][ ][ ][ ][ ] + * + * These nodes may hold samples for query C: + * [ ][ ][ ][ ][ ][ C ][ C ][ C ][ ] + * + * The illustration assumes we have an even distribution of periodic + * samples so all nodes have the same size plotted against time: + * + * Note, to simplify code, the list is never empty. + * + * With overlapping queries we can see that periodic OA reports may + * relate to multiple queries and care needs to be take to keep + * track of sample buffers until there are no queries that might + * depend on their contents. + * + * We use a node ref counting system where a reference ensures that a + * node and all following nodes can't be freed/recycled until the + * reference drops to zero. + * + * E.g. with a ref of one here: + * [ 0 ][ 0 ][ 1 ][ 0 ][ 0 ][ 0 ][ 0 ][ 0 ][ 0 ] + * + * These nodes could be freed or recycled ("reaped"): + * [ 0 ][ 0 ] + * + * These must be preserved until the leading ref drops to zero: + * [ 1 ][ 0 ][ 0 ][ 0 ][ 0 ][ 0 ][ 0 ] + * + * When a query starts we take a reference on the current tail of + * the list, knowing that no already-buffered samples can possibly + * relate to the newly-started query. A pointer to this node is + * also saved in the query object's ->oa.samples_head. + * + * E.g. starting query A while there are two nodes in .sample_buffers: + * ________________A________ + * | + * + * [ 0 ][ 1 ] + * ^_______ Add a reference and store pointer to node in + * A->oa.samples_head + * + * Moving forward to when the B query starts with no new buffer nodes: + * (for reference, i915 perf reads() are only done when queries finish) + * ________________A_______ + * | ________B___ + * | | + * + * [ 0 ][ 2 ] + * ^_______ Add a reference and store pointer to + * node in B->oa.samples_head + * + * Once a query is finished, after an OA query has become 'Ready', + * once the End OA report has landed and after we we have processed + * all the intermediate periodic samples then we drop the + * ->oa.samples_head reference we took at the start. + * + * So when the B query has finished we have: + * ________________A________ + * | ______B___________ + * | | | + * [ 0 ][ 1 ][ 0 ][ 0 ][ 0 ] + * ^_______ Drop B->oa.samples_head reference + * + * We still can't free these due to the A->oa.samples_head ref: + * [ 1 ][ 0 ][ 0 ][ 0 ] + * + * When the A query finishes: (note there's a new ref for C's samples_head) + * ________________A_________________ + * | | + * | _____C_________ + * | | | + * [ 0 ][ 0 ][ 0 ][ 0 ][ 1 ][ 0 ][ 0 ] + * ^_______ Drop A->oa.samples_head reference + * + * And we can now reap these nodes up to the C->oa.samples_head: + * [ X ][ X ][ X ][ X ] + * keeping -> [ 1 ][ 0 ][ 0 ] + * + * We reap old sample buffers each time we finish processing an OA + * query by iterating the sample_buffers list from the head until we + * find a referenced node and stop. + * + * Reaped buffers move to a perfquery.free_sample_buffers list and + * when we come to read() we first look to recycle a buffer from the + * free_sample_buffers list before allocating a new buffer. + */ +struct brw_oa_sample_buf { + struct exec_node link; + int refcount; + int len; + uint8_t buf[I915_PERF_OA_SAMPLE_SIZE * 10]; +}; + /** * i965 representation of a performance query object. * @@ -68,13 +211,56 @@ struct brw_perf_query_object const struct brw_perf_query_info *query; - struct { - /** - * BO containing starting and ending snapshots for the - * statistics counters. - */ - drm_intel_bo *bo; - } pipeline_stats; + /* See query->kind to know which state below is in use... */ + union { + struct { + + /** + * BO containing OA counter snapshots at query Begin/End time. + */ + drm_intel_bo *bo; + + /** + * The MI_REPORT_PERF_COUNT command lets us specify a unique + * ID that will be reflected in the resulting OA report + * that's written by the GPU. This is the ID we're expecting + * in the begin report and the the end report should be + * @begin_report_id + 1. + */ + int begin_report_id; + + /** + * Reference the head of the brw->perfquery.sample_buffers + * list at the time that the query started (so we only need + * to look at nodes after this point when looking for samples + * related to this query) + * + * (See struct brw_oa_sample_buf description for more details) + */ + struct exec_node *samples_head; + + /** + * Storage for the final accumulated OA counters. + */ + uint64_t accumulator[MAX_OA_REPORT_COUNTERS]; + + /** + * false while in the unaccumulated_elements list, and set to + * true when the final, end MI_RPC snapshot has been + * accumulated. + */ + bool results_accumulated; + + } oa; + + struct { + /** + * BO containing starting and ending snapshots for the + * statistics counters. + */ + drm_intel_bo *bo; + } pipeline_stats; + }; }; /** Downcasting convenience macro. */ @@ -88,15 +274,32 @@ brw_perf_query(struct gl_perf_query_object *o) #define STATS_BO_END_OFFSET_BYTES (STATS_BO_SIZE / 2) #define MAX_STAT_COUNTERS (STATS_BO_END_OFFSET_BYTES / 8) +#define MI_RPC_BO_SIZE 4096 +#define MI_RPC_BO_END_OFFSET_BYTES (MI_RPC_BO_SIZE / 2) + /******************************************************************************/ +static bool +brw_is_perf_query_ready(struct gl_context *ctx, + struct gl_perf_query_object *o); + static void dump_perf_query_callback(GLuint id, void *query_void, void *brw_void) { + struct gl_context *ctx = brw_void; struct gl_perf_query_object *o = query_void; struct brw_perf_query_object *obj = query_void; switch (obj->query->kind) { + case OA_COUNTERS: + DBG("%4d: %-6s %-8s BO: %-4s OA data: %-10s %-15s\n", + id, + o->Used ? "Dirty," : "New,", + o->Active ? "Active," : (o->Ready ? "Ready," : "Pending,"), + obj->oa.bo ? "yes," : "no,", + brw_is_perf_query_ready(ctx, o) ? "ready," : "not ready,", + obj->oa.results_accumulated ? "accumulated" : "not accumulated"); + break; case PIPELINE_STATS: DBG("%4d: %-6s %-8s BO: %-4s\n", id, @@ -111,13 +314,68 @@ static void dump_perf_queries(struct brw_context *brw) { struct gl_context *ctx = &brw->ctx; - DBG("Queries: (Open queries = %d)\n", - brw->perfquery.n_active_pipeline_stats_queries); + DBG("Queries: (Open queries = %d, OA users = %d)\n", + brw->perfquery.n_active_oa_queries, brw->perfquery.n_oa_users); _mesa_HashWalk(ctx->PerfQuery.Objects, dump_perf_query_callback, brw); } /******************************************************************************/ +static struct brw_oa_sample_buf * +get_free_sample_buf(struct brw_context *brw) +{ + struct exec_node *node = exec_list_pop_head(&brw->perfquery.free_sample_buffers); + struct brw_oa_sample_buf *buf; + + if (node) + buf = exec_node_data(struct brw_oa_sample_buf, node, link); + else { + buf = ralloc_size(brw, sizeof(*buf)); + + exec_node_init(&buf->link); + buf->refcount = 0; + buf->len = 0; + } + + return buf; +} + +static void +reap_old_sample_buffers(struct brw_context *brw) +{ + struct exec_node *tail_node = + exec_list_get_tail(&brw->perfquery.sample_buffers); + struct brw_oa_sample_buf *tail_buf = + exec_node_data(struct brw_oa_sample_buf, tail_node, link); + + /* Remove all old, unreferenced sample buffers walking forward from + * the head of the list, except always leave at least one node in + * the list so we always have a node to reference when we Begin + * a new query. + */ + foreach_list_typed_safe(struct brw_oa_sample_buf, buf, link, + &brw->perfquery.sample_buffers) + { + if (buf->refcount == 0 && buf != tail_buf) { + exec_node_remove(&buf->link); + exec_list_push_head(&brw->perfquery.free_sample_buffers, &buf->link); + } else + return; + } +} + +static void +free_sample_bufs(struct brw_context *brw) +{ + foreach_list_typed_safe(struct brw_oa_sample_buf, buf, link, + &brw->perfquery.free_sample_buffers) + ralloc_free(buf); + + exec_list_make_empty(&brw->perfquery.free_sample_buffers); +} + +/******************************************************************************/ + /** * Driver hook for glGetPerfQueryInfoINTEL(). */ @@ -138,6 +396,10 @@ brw_get_perf_query_info(struct gl_context *ctx, *n_counters = query->n_counters; switch (query->kind) { + case OA_COUNTERS: + *n_active = brw->perfquery.n_active_oa_queries; + break; + case PIPELINE_STATS: *n_active = brw->perfquery.n_active_pipeline_stats_queries; break; @@ -199,6 +461,397 @@ snapshot_statistics_registers(struct brw_context *brw, } } +/** + * Emit an MI_REPORT_PERF_COUNT command packet. + * + * This asks the GPU to write a report of the current OA counter + * values into @bo at the given offset and containing the given + * @report_id which we can cross-reference when parsing the report. + */ +static void +emit_mi_report_perf_count(struct brw_context *brw, + drm_intel_bo *bo, + uint32_t offset_in_bytes, + uint32_t report_id) +{ + assert(offset_in_bytes % 64 == 0); + + BEGIN_BATCH(3); + OUT_BATCH(GEN6_MI_REPORT_PERF_COUNT); + OUT_RELOC(bo, I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION, + offset_in_bytes); + OUT_BATCH(report_id); + ADVANCE_BATCH(); +} + +/** + * Add a query to the global list of "unaccumulated queries." + * + * Queries are tracked here until all the associated OA reports have + * been accumulated via accumulate_oa_reports() after the end + * MI_REPORT_PERF_COUNT has landed in query->oa.bo. + */ +static void +add_to_unaccumulated_query_list(struct brw_context *brw, + struct brw_perf_query_object *obj) +{ + if (brw->perfquery.unaccumulated_elements >= + brw->perfquery.unaccumulated_array_size) + { + brw->perfquery.unaccumulated_array_size *= 1.5; + brw->perfquery.unaccumulated = + reralloc(brw, brw->perfquery.unaccumulated, + struct brw_perf_query_object *, + brw->perfquery.unaccumulated_array_size); + } + + brw->perfquery.unaccumulated[brw->perfquery.unaccumulated_elements++] = obj; +} + +/** + * Remove a query from the global list of unaccumulated queries once + * after successfully accumulating the OA reports associated with the + * query in accumulate_oa_reports() or when discarding unwanted query + * results. + */ +static void +drop_from_unaccumulated_query_list(struct brw_context *brw, + struct brw_perf_query_object *obj) +{ + for (int i = 0; i < brw->perfquery.unaccumulated_elements; i++) { + if (brw->perfquery.unaccumulated[i] == obj) { + int last_elt = --brw->perfquery.unaccumulated_elements; + + if (i == last_elt) + brw->perfquery.unaccumulated[i] = NULL; + else { + brw->perfquery.unaccumulated[i] = + brw->perfquery.unaccumulated[last_elt]; + } + + break; + } + } + + /* Drop our samples_head reference so that associated periodic + * sample data buffers can potentially be reaped if they aren't + * referenced by any other queries... + */ + + struct brw_oa_sample_buf *buf = + exec_node_data(struct brw_oa_sample_buf, obj->oa.samples_head, link); + + assert(buf->refcount > 0); + buf->refcount--; + + obj->oa.samples_head = NULL; + + reap_old_sample_buffers(brw); +} + +static uint64_t +timebase_scale(struct brw_context *brw, uint32_t u32_time_delta) +{ + uint64_t tmp = ((uint64_t)u32_time_delta) * 1000000000ull; + + return tmp ? tmp / brw->perfquery.sys_vars.timestamp_frequency : 0; +} + +static void +accumulate_uint32(const uint32_t *report0, + const uint32_t *report1, + uint64_t *accumulator) +{ + *accumulator += (uint32_t)(*report1 - *report0); +} + +/** + * Given pointers to starting and ending OA snapshots, add the deltas for each + * counter to the results. + */ +static void +add_deltas(struct brw_context *brw, + struct brw_perf_query_object *obj, + const uint32_t *start, + const uint32_t *end) +{ + const struct brw_perf_query_info *query = obj->query; + uint64_t *accumulator = obj->oa.accumulator; + int i; + + switch (query->oa_format) { + case I915_OA_FORMAT_A45_B8_C8: + accumulate_uint32(start + 1, end + 1, accumulator); /* timestamp */ + + for (i = 0; i < 61; i++) + accumulate_uint32(start + 3 + i, end + 3 + i, accumulator + 1 + i); + + break; + default: + unreachable("Can't accumulate OA counters in unknown format"); + } +} + +static bool +inc_n_oa_users(struct brw_context *brw) +{ + if (brw->perfquery.n_oa_users == 0 && + drmIoctl(brw->perfquery.oa_stream_fd, + I915_PERF_IOCTL_ENABLE, 0) < 0) + { + return false; + } + ++brw->perfquery.n_oa_users; + + return true; +} + +static void +dec_n_oa_users(struct brw_context *brw) +{ + /* Disabling the i915 perf stream will effectively disable the OA + * counters. Note it's important to be sure there are no outstanding + * MI_RPC commands at this point since they could stall the CS + * indefinitely once OACONTROL is disabled. + */ + --brw->perfquery.n_oa_users; + if (brw->perfquery.n_oa_users == 0 && + drmIoctl(brw->perfquery.oa_stream_fd, I915_PERF_IOCTL_DISABLE, 0) < 0) + { + DBG("WARNING: Error disabling i915 perf stream: %m\n"); + } +} + +/* In general if we see anything spurious while accumulating results, + * we don't try and continue accumulating the current query, hoping + * for the best, we scrap anything outstanding, and then hope for the + * best with new queries. + */ +static void +discard_all_queries(struct brw_context *brw) +{ + while (brw->perfquery.unaccumulated_elements) { + struct brw_perf_query_object *obj = brw->perfquery.unaccumulated[0]; + + obj->oa.results_accumulated = true; + drop_from_unaccumulated_query_list(brw, brw->perfquery.unaccumulated[0]); + + dec_n_oa_users(brw); + } +} + +static bool +read_oa_samples(struct brw_context *brw) +{ + while (1) { + struct brw_oa_sample_buf *buf = get_free_sample_buf(brw); + int len; + + while ((len = read(brw->perfquery.oa_stream_fd, buf->buf, + sizeof(buf->buf))) < 0 && errno == EINTR) + ; + + if (len <= 0) { + exec_list_push_tail(&brw->perfquery.free_sample_buffers, &buf->link); + + if (len < 0) { + if (errno == EAGAIN) + return true; + else { + DBG("Error reading i915 perf samples: %m\n"); + return false; + } + } else { + DBG("Spurious EOF reading i915 perf samples\n"); + return false; + } + } + + buf->len = len; + exec_list_push_tail(&brw->perfquery.sample_buffers, &buf->link); + } + + unreachable("not reached"); + return false; +} + +/** + * Accumulate raw OA counter values based on deltas between pairs + * of OA reports. + * + * Accumulation starts from the first report captured via + * MI_REPORT_PERF_COUNT (MI_RPC) by brw_begin_perf_query() until the + * last MI_RPC report requested by brw_end_perf_query(). Between these + * two reports there may also some number of periodically sampled OA + * reports collected via the i915 perf interface - depending on the + * duration of the query. + * + * These periodic snapshots help to ensure we handle counter overflow + * correctly by being frequent enough to ensure we don't miss multiple + * overflows of a counter between snapshots. + */ +static void +accumulate_oa_reports(struct brw_context *brw, + struct brw_perf_query_object *obj) +{ + struct gl_perf_query_object *o = &obj->base; + uint32_t *query_buffer; + uint32_t *start; + uint32_t *last; + uint32_t *end; + struct exec_node *first_samples_node; + + assert(o->Ready); + + /* Collect the latest periodic OA reports from i915 perf */ + if (!read_oa_samples(brw)) + goto error; + + drm_intel_bo_map(obj->oa.bo, false); + query_buffer = obj->oa.bo->virtual; + + start = last = query_buffer; + end = query_buffer + (MI_RPC_BO_END_OFFSET_BYTES / sizeof(uint32_t)); + + if (start[0] != obj->oa.begin_report_id) { + DBG("Spurious start report id=%"PRIu32"\n", start[0]); + goto error; + } + if (end[0] != (obj->oa.begin_report_id + 1)) { + DBG("Spurious end report id=%"PRIu32"\n", end[0]); + goto error; + } + + /* See if we have any periodic reports to accumulate too... */ + + /* N.B. The oa.samples_head was set when the query began and + * pointed to the tail of the brw->perfquery.sample_buffers list at + * the time the query started. Since the buffer existed before the + * first MI_REPORT_PERF_COUNT command was emitted we therefore know + * that no data in this particular node's buffer can possibly be + * associated with the query - so skip ahead one... + */ + first_samples_node = obj->oa.samples_head->next; + + foreach_list_typed_from(struct brw_oa_sample_buf, buf, link, + &brw->perfquery.sample_buffers, + first_samples_node) + { + int offset = 0; + + while (offset < buf->len) { + const struct drm_i915_perf_record_header *header = + (const struct drm_i915_perf_record_header *)(buf->buf + offset); + + assert(header->size != 0); + assert(header->size <= buf->len); + + offset += header->size; + + switch (header->type) { + case DRM_I915_PERF_RECORD_SAMPLE: { + uint32_t *report = (uint32_t *)(header + 1); + + /* Ignore reports that come before the start marker. + * (Note: takes care to allow overflow of 32bit timestamps) + */ + if (timebase_scale(brw, report[1] - start[1]) > 5000000000) + continue; + + /* Ignore reports that come after the end marker. + * (Note: takes care to allow overflow of 32bit timestamps) + */ + if (timebase_scale(brw, report[1] - end[1]) <= 5000000000) + goto end; + + add_deltas(brw, obj, last, report); + + last = report; + + break; + } + + case DRM_I915_PERF_RECORD_OA_BUFFER_LOST: + DBG("i915 perf: OA error: all reports lost\n"); + goto error; + case DRM_I915_PERF_RECORD_OA_REPORT_LOST: + DBG("i915 perf: OA report lost\n"); + break; + } + } + } + +end: + + add_deltas(brw, obj, last, end); + + DBG("Marking %d accumulated - results gathered\n", o->Id); + + drm_intel_bo_unmap(obj->oa.bo); + obj->oa.results_accumulated = true; + drop_from_unaccumulated_query_list(brw, obj); + dec_n_oa_users(brw); + + return; + +error: + + drm_intel_bo_unmap(obj->oa.bo); + discard_all_queries(brw); +} + +/******************************************************************************/ + +static bool +open_i915_perf_oa_stream(struct brw_context *brw, + int metrics_set_id, + int report_format, + int period_exponent, + int drm_fd, + uint32_t ctx_id) +{ + uint64_t properties[] = { + /* Single context sampling */ + DRM_I915_PERF_PROP_CTX_HANDLE, ctx_id, + + /* Include OA reports in samples */ + DRM_I915_PERF_PROP_SAMPLE_OA, true, + + /* OA unit configuration */ + DRM_I915_PERF_PROP_OA_METRICS_SET, metrics_set_id, + DRM_I915_PERF_PROP_OA_FORMAT, report_format, + DRM_I915_PERF_PROP_OA_EXPONENT, period_exponent, + }; + struct drm_i915_perf_open_param param = { + .flags = I915_PERF_FLAG_FD_CLOEXEC | + I915_PERF_FLAG_FD_NONBLOCK | + I915_PERF_FLAG_DISABLED, + .num_properties = ARRAY_SIZE(properties) / 2, + .properties_ptr = (uint64_t)properties + }; + int fd = drmIoctl(drm_fd, DRM_IOCTL_I915_PERF_OPEN, ¶m); + if (fd == -1) { + DBG("Error opening i915 perf OA stream: %m\n"); + return false; + } + + brw->perfquery.oa_stream_fd = fd; + + brw->perfquery.current_oa_metrics_set_id = metrics_set_id; + brw->perfquery.current_oa_format = report_format; + + return true; +} + +static void +close_perf(struct brw_context *brw) +{ + if (brw->perfquery.oa_stream_fd != -1) { + close(brw->perfquery.oa_stream_fd); + brw->perfquery.oa_stream_fd = -1; + } +} + /** * Driver hook for glBeginPerfQueryINTEL(). */ @@ -268,6 +921,112 @@ brw_begin_perf_query(struct gl_context *ctx, brw_emit_mi_flush(brw); switch (query->kind) { + case OA_COUNTERS: + + /* Opening an i915 perf stream implies exclusive access to the OA unit + * which will generate counter reports for a specific counter set with a + * specific layout/format so we can't begin any OA based queries that + * require a different counter set or format unless we get an opportunity + * to close the stream and open a new one... + */ + if (brw->perfquery.oa_stream_fd != -1 && + brw->perfquery.current_oa_metrics_set_id != + query->oa_metrics_set_id) { + + if (brw->perfquery.n_oa_users != 0) + return false; + else + close_perf(brw); + } + + /* If the OA counters aren't already on, enable them. */ + if (brw->perfquery.oa_stream_fd == -1) { + __DRIscreen *screen = brw->screen->driScrnPriv; + uint32_t ctx_id; + int period_exponent; + + if (drm_intel_gem_context_get_id(brw->hw_ctx, &ctx_id) != 0) + return false; + + /* The timestamp for HSW+ increments every 80ns + * + * The period_exponent gives a sampling period as follows: + * sample_period = 80ns * 2^(period_exponent + 1) + * + * The overflow period for Haswell can be calculated as: + * + * 2^32 / (n_eus * max_gen_freq * 2) + * (E.g. 40 EUs @ 1GHz = ~53ms) + * + * We currently sample every 42 milliseconds... + */ + period_exponent = 18; + + if (!open_i915_perf_oa_stream(brw, + query->oa_metrics_set_id, + query->oa_format, + period_exponent, + screen->fd, /* drm fd */ + ctx_id)) + return false; + } else { + assert(brw->perfquery.current_oa_metrics_set_id == + query->oa_metrics_set_id && + brw->perfquery.current_oa_format == + query->oa_format); + } + + if (!inc_n_oa_users(brw)) { + DBG("WARNING: Error enabling i915 perf stream: %m\n"); + return false; + } + + if (obj->oa.bo) { + drm_intel_bo_unreference(obj->oa.bo); + obj->oa.bo = NULL; + } + + obj->oa.bo = + drm_intel_bo_alloc(brw->bufmgr, "perf. query OA MI_RPC bo", + MI_RPC_BO_SIZE, 64); +#ifdef DEBUG + /* Pre-filling the BO helps debug whether writes landed. */ + drm_intel_bo_map(obj->oa.bo, true); + memset((char *) obj->oa.bo->virtual, 0x80, MI_RPC_BO_SIZE); + drm_intel_bo_unmap(obj->oa.bo); +#endif + + obj->oa.begin_report_id = brw->perfquery.next_query_start_report_id; + brw->perfquery.next_query_start_report_id += 2; + + /* Take a starting OA counter snapshot. */ + emit_mi_report_perf_count(brw, obj->oa.bo, 0, + obj->oa.begin_report_id); + ++brw->perfquery.n_active_oa_queries; + + /* No already-buffered samples can possibly be associated with this query + * so create a marker within the list of sample buffers enabling us to + * easily ignore earlier samples when processing this query after + * completion. + */ + assert(!exec_list_is_empty(&brw->perfquery.sample_buffers)); + obj->oa.samples_head = exec_list_get_tail(&brw->perfquery.sample_buffers); + + struct brw_oa_sample_buf *buf = + exec_node_data(struct brw_oa_sample_buf, obj->oa.samples_head, link); + + /* This reference will ensure that future/following sample + * buffers (that may relate to this query) can't be freed until + * this drops to zero. + */ + buf->refcount++; + + memset(obj->oa.accumulator, 0, sizeof(obj->oa.accumulator)); + obj->oa.results_accumulated = false; + + add_to_unaccumulated_query_list(brw, obj); + break; + case PIPELINE_STATS: if (obj->pipeline_stats.bo) { drm_intel_bo_unreference(obj->pipeline_stats.bo); @@ -312,6 +1071,28 @@ brw_end_perf_query(struct gl_context *ctx, brw_emit_mi_flush(brw); switch (obj->query->kind) { + case OA_COUNTERS: + + /* NB: It's possible that the query will have already been marked + * as 'accumulated' if an error was seen while reading samples + * from perf. In this case we mustn't try and emit a closing + * MI_RPC command in case the OA unit has already been disabled + */ + if (!obj->oa.results_accumulated) { + /* Take an ending OA counter snapshot. */ + emit_mi_report_perf_count(brw, obj->oa.bo, + MI_RPC_BO_END_OFFSET_BYTES, + obj->oa.begin_report_id + 1); + } + + --brw->perfquery.n_active_oa_queries; + + /* NB: even though the query has now ended, it can't be accumulated + * until the end MI_REPORT_PERF_COUNT snapshot has been written + * to query->oa.bo + */ + break; + case PIPELINE_STATS: snapshot_statistics_registers(brw, obj, STATS_BO_END_OFFSET_BYTES); @@ -330,6 +1111,10 @@ brw_wait_perf_query(struct gl_context *ctx, struct gl_perf_query_object *o) assert(!o->Ready); switch (obj->query->kind) { + case OA_COUNTERS: + bo = obj->oa.bo; + break; + case PIPELINE_STATS: bo = obj->pipeline_stats.bo; break; @@ -339,7 +1124,8 @@ brw_wait_perf_query(struct gl_context *ctx, struct gl_perf_query_object *o) return; /* If the current batch references our results bo then we need to - * flush first... */ + * flush first... + */ if (drm_intel_bo_references(brw->batch.bo, bo)) intel_batchbuffer_flush(brw); @@ -362,6 +1148,12 @@ brw_is_perf_query_ready(struct gl_context *ctx, return true; switch (obj->query->kind) { + case OA_COUNTERS: + return (obj->oa.results_accumulated || + (obj->oa.bo && + !drm_intel_bo_references(brw->batch.bo, obj->oa.bo) && + !drm_intel_bo_busy(obj->oa.bo))); + case PIPELINE_STATS: return (obj->pipeline_stats.bo && !drm_intel_bo_references(brw->batch.bo, obj->pipeline_stats.bo) && @@ -372,6 +1164,49 @@ brw_is_perf_query_ready(struct gl_context *ctx, return false; } +static int +get_oa_counter_data(struct brw_context *brw, + struct brw_perf_query_object *obj, + size_t data_size, + uint8_t *data) +{ + const struct brw_perf_query_info *query = obj->query; + int n_counters = query->n_counters; + int written = 0; + + if (!obj->oa.results_accumulated) { + accumulate_oa_reports(brw, obj); + assert(obj->oa.results_accumulated); + } + + for (int i = 0; i < n_counters; i++) { + const struct brw_perf_query_counter *counter = &query->counters[i]; + uint64_t *out_uint64; + float *out_float; + + if (counter->size) { + switch (counter->data_type) { + case GL_PERFQUERY_COUNTER_DATA_UINT64_INTEL: + out_uint64 = (uint64_t *)(data + counter->offset); + *out_uint64 = counter->oa_counter_read_uint64(brw, query, + obj->oa.accumulator); + break; + case GL_PERFQUERY_COUNTER_DATA_FLOAT_INTEL: + out_float = (float *)(data + counter->offset); + *out_float = counter->oa_counter_read_float(brw, query, + obj->oa.accumulator); + break; + default: + /* So far we aren't using uint32, double or bool32... */ + unreachable("unexpected counter data type"); + } + written = counter->offset + counter->size; + } + } + + return written; +} + static int get_pipeline_stats_data(struct brw_context *brw, struct brw_perf_query_object *obj, @@ -433,6 +1268,10 @@ brw_get_perf_query_data(struct gl_context *ctx, assert(o->Ready); switch (obj->query->kind) { + case OA_COUNTERS: + written = get_oa_counter_data(brw, obj, data_size, (uint8_t *)data); + break; + case PIPELINE_STATS: written = get_pipeline_stats_data(brw, obj, data_size, (uint8_t *)data); break; @@ -456,6 +1295,8 @@ brw_new_perf_query_object(struct gl_context *ctx, unsigned query_index) obj->query = query; + brw->perfquery.n_query_instances++; + return &obj->base; } @@ -466,6 +1307,7 @@ static void brw_delete_perf_query(struct gl_context *ctx, struct gl_perf_query_object *o) { + struct brw_context *brw = brw_context(ctx); struct brw_perf_query_object *obj = brw_perf_query(o); /* We can assume that the frontend waits for a query to complete @@ -478,6 +1320,20 @@ brw_delete_perf_query(struct gl_context *ctx, DBG("Delete(%d)\n", o->Id); switch (obj->query->kind) { + case OA_COUNTERS: + if (obj->oa.bo) { + if (!obj->oa.results_accumulated) { + drop_from_unaccumulated_query_list(brw, obj); + dec_n_oa_users(brw); + } + + drm_intel_bo_unreference(obj->oa.bo); + obj->oa.bo = NULL; + } + + obj->oa.results_accumulated = false; + break; + case PIPELINE_STATS: if (obj->pipeline_stats.bo) { drm_intel_bo_unreference(obj->pipeline_stats.bo); @@ -487,6 +1343,15 @@ brw_delete_perf_query(struct gl_context *ctx, } free(obj); + + /* As an indication that the INTEL_performance_query extension is no + * longer in use, it's a good time to free our cache of sample + * buffers and close any current i915-perf stream. + */ + if (--brw->perfquery.n_query_instances == 0) { + free_sample_bufs(brw); + close_perf(brw); + } } /******************************************************************************/ @@ -618,16 +1483,244 @@ init_pipeline_statistic_query_registers(struct brw_context *brw) query->data_size = sizeof(uint64_t) * query->n_counters; } +static bool +read_file_uint64(const char *file, uint64_t *val) +{ + char buf[32]; + int fd, n; + + fd = open(file, 0); + if (fd < 0) + return false; + n = read(fd, buf, sizeof (buf) - 1); + close(fd); + if (n < 0) + return false; + + buf[n] = '\0'; + *val = strtoull(buf, NULL, 0); + + return true; +} + +static void +enumerate_sysfs_metrics(struct brw_context *brw, const char *sysfs_dev_dir) +{ + char buf[256]; + DIR *metricsdir = NULL; + struct dirent *metric_entry; + + strncpy(buf, sysfs_dev_dir, sizeof(buf)); + strncat(buf, "/metrics", sizeof(buf)); + + metricsdir = opendir(buf); + if (!metricsdir) { + DBG("Failed to open %s: %m\n", buf); + return; + } + + while ((metric_entry = readdir(metricsdir))) { + struct hash_entry *entry; + + if ((metric_entry->d_type != DT_DIR && + metric_entry->d_type != DT_LNK) || + metric_entry->d_name[0] == '.') + continue; + + DBG("metric set: %s\n", metric_entry->d_name); + entry = _mesa_hash_table_search(brw->perfquery.oa_metrics_table, + metric_entry->d_name); + if (entry) { + struct brw_perf_query_info *query; + uint64_t id; + + snprintf(buf, sizeof(buf), "%s/metrics/%s/id", + sysfs_dev_dir, metric_entry->d_name); + + if (!read_file_uint64(buf, &id)) { + DBG("Failed to read metric set id from %s: %m", buf); + continue; + } + + query = append_query_info(brw); + *query = *(struct brw_perf_query_info *)entry->data; + query->oa_metrics_set_id = id; + + DBG("metric set known by mesa: id = %" PRIu64"\n", + query->oa_metrics_set_id); + } else + DBG("metric set not known by mesa (skipping)\n"); + } + + closedir(metricsdir); +} + +static bool +read_sysfs_drm_device_file_uint64(struct brw_context *brw, + const char *sysfs_dev_dir, + const char *file, + uint64_t *value) +{ + char buf[512]; + + snprintf(buf, sizeof(buf), "%s/%s", sysfs_dev_dir, file); + + return read_file_uint64(buf, value); +} + +static bool +init_oa_sys_vars(struct brw_context *brw, const char *sysfs_dev_dir) +{ + uint64_t min_freq_mhz = 0, max_freq_mhz = 0; + + if (!read_sysfs_drm_device_file_uint64(brw, sysfs_dev_dir, + "gt_min_freq_mhz", + &min_freq_mhz)) + return false; + + if (!read_sysfs_drm_device_file_uint64(brw, sysfs_dev_dir, + "gt_max_freq_mhz", + &max_freq_mhz)) + return false; + + brw->perfquery.sys_vars.gt_min_freq = min_freq_mhz * 1000000; + brw->perfquery.sys_vars.gt_max_freq = max_freq_mhz * 1000000; + + if (brw->is_haswell) { + const struct gen_device_info *info = &brw->screen->devinfo; + + brw->perfquery.sys_vars.timestamp_frequency = 12500000; + + if (info->gt == 1) { + brw->perfquery.sys_vars.n_eus = 10; + brw->perfquery.sys_vars.n_eu_slices = 1; + brw->perfquery.sys_vars.subslice_mask = 0x1; + } else if (info->gt == 2) { + brw->perfquery.sys_vars.n_eus = 20; + brw->perfquery.sys_vars.n_eu_slices = 1; + brw->perfquery.sys_vars.subslice_mask = 0x3; + } else if (info->gt == 3) { + brw->perfquery.sys_vars.n_eus = 40; + brw->perfquery.sys_vars.n_eu_slices = 2; + brw->perfquery.sys_vars.subslice_mask = 0xf; + } else + unreachable("not reached"); + + return true; + } else + return false; +} + +static bool +get_sysfs_dev_dir(struct brw_context *brw, + char *path_buf, + int path_buf_len) +{ + __DRIscreen *screen = brw->screen->driScrnPriv; + struct stat sb; + int min, maj; + DIR *drmdir; + struct dirent *drm_entry; + + assert(path_buf); + assert(path_buf_len); + path_buf[0] = '\0'; + + if (fstat(screen->fd, &sb)) { + DBG("Failed to stat DRM fd\n"); + return false; + } + + maj = major(sb.st_rdev); + min = minor(sb.st_rdev); + + if (!S_ISCHR(sb.st_mode)) { + DBG("DRM fd is not a character device as expected\n"); + return false; + } + + snprintf(path_buf, path_buf_len, + "/sys/dev/char/%d:%d/device/drm", maj, min); + + drmdir = opendir(path_buf); + if (!drmdir) { + DBG("Failed to open %s: %m\n", path_buf); + return false; + } + + while ((drm_entry = readdir(drmdir))) { + if ((drm_entry->d_type == DT_DIR || + drm_entry->d_type == DT_LNK) && + strncmp(drm_entry->d_name, "card", 4) == 0) + { + snprintf(path_buf, path_buf_len, + "/sys/dev/char/%d:%d/device/drm/%s", + maj, min, drm_entry->d_name); + closedir(drmdir); + return true; + } + } + + closedir(drmdir); + + DBG("Failed to find cardX directory under /sys/dev/char/%d:%d/device/drm\n", + maj, min); + + return false; +} + static unsigned brw_init_perf_query_info(struct gl_context *ctx) { struct brw_context *brw = brw_context(ctx); + struct stat sb; + char sysfs_dev_dir[128]; if (brw->perfquery.n_queries) return brw->perfquery.n_queries; init_pipeline_statistic_query_registers(brw); + /* The existence of this sysctl parameter implies the kernel supports + * the i915 perf interface. + */ + if (brw->is_haswell && + stat("/proc/sys/dev/i915/perf_stream_paranoid", &sb) == 0 && + get_sysfs_dev_dir(brw, sysfs_dev_dir, sizeof(sysfs_dev_dir)) && + init_oa_sys_vars(brw, sysfs_dev_dir)) + { + brw->perfquery.oa_metrics_table = + _mesa_hash_table_create(NULL, _mesa_key_hash_string, + _mesa_key_string_equal); + + /* Index all the metric sets mesa knows about before looking to + * see what the kernel is advertising. + */ + brw_oa_register_queries_hsw(brw); + + enumerate_sysfs_metrics(brw, sysfs_dev_dir); + } + + brw->perfquery.unaccumulated = + ralloc_array(brw, struct brw_perf_query_object *, 2); + brw->perfquery.unaccumulated_elements = 0; + brw->perfquery.unaccumulated_array_size = 2; + + exec_list_make_empty(&brw->perfquery.sample_buffers); + exec_list_make_empty(&brw->perfquery.free_sample_buffers); + + /* It's convenient to guarantee that this linked list of sample + * buffers is never empty so we add an empty head so when we + * Begin an OA query we can always take a reference on a buffer + * in this list. + */ + struct brw_oa_sample_buf *buf = get_free_sample_buf(brw); + exec_list_push_head(&brw->perfquery.sample_buffers, &buf->link); + + brw->perfquery.oa_stream_fd = -1; + + brw->perfquery.next_query_start_report_id = 1000; + return brw->perfquery.n_queries; }