src/gallium/drivers/freedreno/freedreno_batch_cache.c - third_party/mesa - Git at Google

 /*
  * Copyright (C) 2016 Rob Clark <robclark@freedesktop.org>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
  * to deal in the Software without restriction, including without limitation
  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  * and/or sell copies of the Software, and to permit persons to whom the
  * Software is furnished to do so, subject to the following conditions:
  *
  * The above copyright notice and this permission notice (including the next
  * paragraph) shall be included in all copies or substantial portions of the
  * Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  *
  * Authors:
  *    Rob Clark <robclark@freedesktop.org>
  */

 #include "util/hash_table.h"
 #include "util/set.h"
 #include "util/list.h"
 #include "util/u_string.h"

 #include "freedreno_batch.h"
 #include "freedreno_batch_cache.h"
 #include "freedreno_context.h"
 #include "freedreno_resource.h"

 /* Overview:
  *
  *   The batch cache provides lookup for mapping pipe_framebuffer_state
  *   to a batch.
  *
  *   It does this via hashtable, with key that roughly matches the
  *   pipe_framebuffer_state, as described below.
  *
  * Batch Cache hashtable key:
  *
  *   To serialize the key, and to avoid dealing with holding a reference to
  *   pipe_surface's (which hold a reference to pipe_resource and complicate
  *   the whole refcnting thing), the key is variable length and inline's the
  *   pertinent details of the pipe_surface.
  *
  * Batch:
  *
  *   Each batch needs to hold a reference to each resource it depends on (ie.
  *   anything that needs a mem2gmem).  And a weak reference to resources it
  *   renders to.  (If both src[n] and dst[n] are not NULL then they are the
  *   same.)
  *
  *   When a resource is destroyed, we need to remove entries in the batch
  *   cache that reference the resource, to avoid dangling pointer issues.
  *   So each resource holds a hashset of batches which have reference them
  *   in their hashtable key.
  *
  *   When a batch has weak reference to no more resources (ie. all the
  *   surfaces it rendered to are destroyed) the batch can be destroyed.
  *   Could happen in an app that renders and never uses the result.  More
  *   common scenario, I think, will be that some, but not all, of the
  *   surfaces are destroyed before the batch is submitted.
  *
  *   If (for example), batch writes to zsbuf but that surface is destroyed
  *   before batch is submitted, we can skip gmem2mem (but still need to
  *   alloc gmem space as before.  If the batch depended on previous contents
  *   of that surface, it would be holding a reference so the surface would
  *   not have been destroyed.
  */

 struct key {
 	uint32_t width, height, layers;
 	uint16_t samples, num_surfs;
 	struct fd_context *ctx;
 	struct {
 		struct pipe_resource *texture;
 		union pipe_surface_desc u;
 		uint16_t pos, format;
 	} surf[0];
 };

 static struct key *
 key_alloc(unsigned num_surfs)
 {
 	struct key *key =
 		CALLOC_VARIANT_LENGTH_STRUCT(key, sizeof(key->surf[0]) * num_surfs);
 	return key;
 }

 static uint32_t
 key_hash(const void *_key)
 {
 	const struct key *key = _key;
 	uint32_t hash = _mesa_fnv32_1a_offset_bias;
 	hash = _mesa_fnv32_1a_accumulate_block(hash, key, offsetof(struct key, surf[0]));
 	hash = _mesa_fnv32_1a_accumulate_block(hash, key->surf, sizeof(key->surf[0]) * key->num_surfs);
 	return hash;
 }

 static bool
 key_equals(const void *_a, const void *_b)
 {
 	const struct key *a = _a;
 	const struct key *b = _b;
 	return (memcmp(a, b, offsetof(struct key, surf[0])) == 0) &&
 		(memcmp(a->surf, b->surf, sizeof(a->surf[0]) * a->num_surfs) == 0);
 }

 void
 fd_bc_init(struct fd_batch_cache *cache)
 {
 	cache->ht = _mesa_hash_table_create(NULL, key_hash, key_equals);
 }

 void
 fd_bc_fini(struct fd_batch_cache *cache)
 {
 	_mesa_hash_table_destroy(cache->ht, NULL);
 }

 void
 fd_bc_flush(struct fd_batch_cache *cache, struct fd_context *ctx)
 {
 	struct hash_entry *entry;
 	struct fd_batch *last_batch = NULL;

 	mtx_lock(&ctx->screen->lock);

 	hash_table_foreach(cache->ht, entry) {
 		struct fd_batch *batch = NULL;
 		/* hold a reference since we can drop screen->lock: */
 		fd_batch_reference_locked(&batch, (struct fd_batch *)entry->data);
 		if (batch->ctx == ctx) {
 			mtx_unlock(&ctx->screen->lock);
 			fd_batch_reference(&last_batch, batch);
 			fd_batch_flush(batch, false, false);
 			mtx_lock(&ctx->screen->lock);
 		}
 		fd_batch_reference_locked(&batch, NULL);
 	}

 	mtx_unlock(&ctx->screen->lock);

 	if (last_batch) {
 		fd_batch_sync(last_batch);
 		fd_batch_reference(&last_batch, NULL);
 	}
 }

 /* deferred flush doesn't actually flush, but it marks every other
  * batch associated with the context as dependent on the current
  * batch.  So when the current batch gets flushed, all other batches
  * that came before also get flushed.
  */
 void
 fd_bc_flush_deferred(struct fd_batch_cache *cache, struct fd_context *ctx)
 {
 	struct fd_batch *current_batch = ctx->batch;
 	struct hash_entry *entry;

 	mtx_lock(&ctx->screen->lock);

 	hash_table_foreach(cache->ht, entry) {
 		struct fd_batch *batch = entry->data;
 		if (batch == current_batch)
 			continue;
 		if (batch->ctx == ctx)
 			fd_batch_add_dep(current_batch, batch);
 	}

 	mtx_unlock(&ctx->screen->lock);
 }

 void
 fd_bc_invalidate_context(struct fd_context *ctx)
 {
 	struct fd_batch_cache *cache = &ctx->screen->batch_cache;
 	struct fd_batch *batch;

 	mtx_lock(&ctx->screen->lock);

 	foreach_batch(batch, cache, cache->batch_mask) {
 		if (batch->ctx == ctx)
 			fd_batch_reference_locked(&batch, NULL);
 	}

 	mtx_unlock(&ctx->screen->lock);
 }

 void
 fd_bc_invalidate_batch(struct fd_batch *batch, bool destroy)
 {
 	if (!batch)
 		return;

 	struct fd_batch_cache *cache = &batch->ctx->screen->batch_cache;
 	struct key *key = (struct key *)batch->key;

 	pipe_mutex_assert_locked(batch->ctx->screen->lock);

 	if (destroy) {
 		cache->batches[batch->idx] = NULL;
 		cache->batch_mask &= ~(1 << batch->idx);
 	}

 	if (!key)
 		return;

 	DBG("%p: key=%p", batch, batch->key);
 	for (unsigned idx = 0; idx < key->num_surfs; idx++) {
 		struct fd_resource *rsc = fd_resource(key->surf[idx].texture);
 		rsc->bc_batch_mask &= ~(1 << batch->idx);
 	}

 	struct hash_entry *entry =
 		_mesa_hash_table_search_pre_hashed(cache->ht, batch->hash, key);
 	_mesa_hash_table_remove(cache->ht, entry);

 	batch->key = NULL;
 	free(key);
 }

 void
 fd_bc_invalidate_resource(struct fd_resource *rsc, bool destroy)
 {
 	struct fd_screen *screen = fd_screen(rsc->base.screen);
 	struct fd_batch *batch;

 	mtx_lock(&screen->lock);

 	if (destroy) {
 		foreach_batch(batch, &screen->batch_cache, rsc->batch_mask) {
 			struct set_entry *entry = _mesa_set_search(batch->resources, rsc);
 			_mesa_set_remove(batch->resources, entry);
 		}
 		rsc->batch_mask = 0;

 		fd_batch_reference_locked(&rsc->write_batch, NULL);
 	}

 	foreach_batch(batch, &screen->batch_cache, rsc->bc_batch_mask)
 		fd_bc_invalidate_batch(batch, false);

 	rsc->bc_batch_mask = 0;

 	mtx_unlock(&screen->lock);
 }

 struct fd_batch *
 fd_bc_alloc_batch(struct fd_batch_cache *cache, struct fd_context *ctx)
 {
 	struct fd_batch *batch;
 	uint32_t idx;

 	mtx_lock(&ctx->screen->lock);

 	while ((idx = ffs(~cache->batch_mask)) == 0) {
 #if 0
 		for (unsigned i = 0; i < ARRAY_SIZE(cache->batches); i++) {
 			batch = cache->batches[i];
 			debug_printf("%d: needs_flush=%d, depends:", batch->idx, batch->needs_flush);
 			struct set_entry *entry;
 			set_foreach(batch->dependencies, entry) {
 				struct fd_batch *dep = (struct fd_batch *)entry->key;
 				debug_printf(" %d", dep->idx);
 			}
 			debug_printf("\n");
 		}
 #endif
 		/* TODO: is LRU the better policy?  Or perhaps the batch that
 		 * depends on the fewest other batches?
 		 */
 		struct fd_batch *flush_batch = NULL;
 		for (unsigned i = 0; i < ARRAY_SIZE(cache->batches); i++) {
 			if ((cache->batches[i] == ctx->batch) ||
 					!cache->batches[i]->needs_flush)
 				continue;
 			if (!flush_batch || (cache->batches[i]->seqno < flush_batch->seqno))
 				fd_batch_reference_locked(&flush_batch, cache->batches[i]);
 		}

 		/* we can drop lock temporarily here, since we hold a ref,
 		 * flush_batch won't disappear under us.
 		 */
 		mtx_unlock(&ctx->screen->lock);
 		DBG("%p: too many batches!  flush forced!", flush_batch);
 		fd_batch_flush(flush_batch, true, false);
 		mtx_lock(&ctx->screen->lock);

 		/* While the resources get cleaned up automatically, the flush_batch
 		 * doesn't get removed from the dependencies of other batches, so
 		 * it won't be unref'd and will remain in the table.
 		 *
 		 * TODO maybe keep a bitmask of batches that depend on me, to make
 		 * this easier:
 		 */
 		for (unsigned i = 0; i < ARRAY_SIZE(cache->batches); i++) {
 			struct fd_batch *other = cache->batches[i];
 			if (!other)
 				continue;
 			if (other->dependents_mask & (1 << flush_batch->idx)) {
 				other->dependents_mask &= ~(1 << flush_batch->idx);
 				struct fd_batch *ref = flush_batch;
 				fd_batch_reference_locked(&ref, NULL);
 			}
 		}

 		fd_batch_reference_locked(&flush_batch, NULL);
 	}

 	idx--;              /* bit zero returns 1 for ffs() */

 	batch = fd_batch_create(ctx, false);
 	if (!batch)
 		goto out;

 	batch->seqno = cache->cnt++;
 	batch->idx = idx;
 	cache->batch_mask |= (1 << idx);

 	debug_assert(cache->batches[idx] == NULL);
 	cache->batches[idx] = batch;

 out:
 	mtx_unlock(&ctx->screen->lock);

 	return batch;
 }

 static struct fd_batch *
 batch_from_key(struct fd_batch_cache *cache, struct key *key,
 		struct fd_context *ctx)
 {
 	struct fd_batch *batch = NULL;
 	uint32_t hash = key_hash(key);
 	struct hash_entry *entry =
 		_mesa_hash_table_search_pre_hashed(cache->ht, hash, key);

 	if (entry) {
 		free(key);
 		fd_batch_reference(&batch, (struct fd_batch *)entry->data);
 		return batch;
 	}

 	batch = fd_bc_alloc_batch(cache, ctx);
 #ifdef DEBUG
 	DBG("%p: hash=0x%08x, %ux%u, %u layers, %u samples", batch, hash,
 			key->width, key->height, key->layers, key->samples);
 	for (unsigned idx = 0; idx < key->num_surfs; idx++) {
 		DBG("%p:  surf[%u]: %p (%s) (%u,%u / %u,%u,%u)", batch, key->surf[idx].pos,
 			key->surf[idx].texture, util_format_name(key->surf[idx].format),
 			key->surf[idx].u.buf.first_element, key->surf[idx].u.buf.last_element,
 			key->surf[idx].u.tex.first_layer, key->surf[idx].u.tex.last_layer,
 			key->surf[idx].u.tex.level);
 	}
 #endif
 	if (!batch)
 		return NULL;

 	mtx_lock(&ctx->screen->lock);

 	_mesa_hash_table_insert_pre_hashed(cache->ht, hash, key, batch);
 	batch->key = key;
 	batch->hash = hash;

 	for (unsigned idx = 0; idx < key->num_surfs; idx++) {
 		struct fd_resource *rsc = fd_resource(key->surf[idx].texture);
 		rsc->bc_batch_mask = (1 << batch->idx);
 	}

 	mtx_unlock(&ctx->screen->lock);

 	return batch;
 }

 static void
 key_surf(struct key *key, unsigned idx, unsigned pos, struct pipe_surface *psurf)
 {
 	key->surf[idx].texture = psurf->texture;
 	key->surf[idx].u = psurf->u;
 	key->surf[idx].pos = pos;
 	key->surf[idx].format = psurf->format;
 }

 struct fd_batch *
 fd_batch_from_fb(struct fd_batch_cache *cache, struct fd_context *ctx,
 		const struct pipe_framebuffer_state *pfb)
 {
 	unsigned idx = 0, n = pfb->nr_cbufs + (pfb->zsbuf ? 1 : 0);
 	struct key *key = key_alloc(n);

 	key->width = pfb->width;
 	key->height = pfb->height;
 	key->layers = pfb->layers;
 	key->samples = pfb->samples;
 	key->ctx = ctx;

 	if (pfb->zsbuf)
 		key_surf(key, idx++, 0, pfb->zsbuf);

 	for (unsigned i = 0; i < pfb->nr_cbufs; i++)
 		if (pfb->cbufs[i])
 			key_surf(key, idx++, i + 1, pfb->cbufs[i]);

 	key->num_surfs = idx;

 	return batch_from_key(cache, key, ctx);
 }
	/*
	* Copyright (C) 2016 Rob Clark <robclark@freedesktop.org>
	*
	* Permission is hereby granted, free of charge, to any person obtaining a
	* copy of this software and associated documentation files (the "Software"),
	* to deal in the Software without restriction, including without limitation
	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	* and/or sell copies of the Software, and to permit persons to whom the
	* Software is furnished to do so, subject to the following conditions:
	*
	* The above copyright notice and this permission notice (including the next
	* paragraph) shall be included in all copies or substantial portions of the
	* Software.
	*
	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	* SOFTWARE.
	*
	* Authors:
	* Rob Clark <robclark@freedesktop.org>
	*/

	#include "util/hash_table.h"
	#include "util/set.h"
	#include "util/list.h"
	#include "util/u_string.h"

	#include "freedreno_batch.h"
	#include "freedreno_batch_cache.h"
	#include "freedreno_context.h"
	#include "freedreno_resource.h"

	/* Overview:
	*
	* The batch cache provides lookup for mapping pipe_framebuffer_state
	* to a batch.
	*
	* It does this via hashtable, with key that roughly matches the
	* pipe_framebuffer_state, as described below.
	*
	* Batch Cache hashtable key:
	*
	* To serialize the key, and to avoid dealing with holding a reference to
	* pipe_surface's (which hold a reference to pipe_resource and complicate
	* the whole refcnting thing), the key is variable length and inline's the
	* pertinent details of the pipe_surface.
	*
	* Batch:
	*
	* Each batch needs to hold a reference to each resource it depends on (ie.
	* anything that needs a mem2gmem). And a weak reference to resources it
	* renders to. (If both src[n] and dst[n] are not NULL then they are the
	* same.)
	*
	* When a resource is destroyed, we need to remove entries in the batch
	* cache that reference the resource, to avoid dangling pointer issues.
	* So each resource holds a hashset of batches which have reference them
	* in their hashtable key.
	*
	* When a batch has weak reference to no more resources (ie. all the
	* surfaces it rendered to are destroyed) the batch can be destroyed.
	* Could happen in an app that renders and never uses the result. More
	* common scenario, I think, will be that some, but not all, of the
	* surfaces are destroyed before the batch is submitted.
	*
	* If (for example), batch writes to zsbuf but that surface is destroyed
	* before batch is submitted, we can skip gmem2mem (but still need to
	* alloc gmem space as before. If the batch depended on previous contents
	* of that surface, it would be holding a reference so the surface would
	* not have been destroyed.
	*/

	struct key {
	uint32_t width, height, layers;
	uint16_t samples, num_surfs;
	struct fd_context *ctx;
	struct {
	struct pipe_resource *texture;
	union pipe_surface_desc u;
	uint16_t pos, format;
	} surf[0];
	};

	static struct key *
	key_alloc(unsigned num_surfs)
	{
	struct key *key =
	CALLOC_VARIANT_LENGTH_STRUCT(key, sizeof(key->surf[0]) * num_surfs);
	return key;
	}

	static uint32_t
	key_hash(const void *_key)
	{
	const struct key *key = _key;
	uint32_t hash = _mesa_fnv32_1a_offset_bias;
	hash = _mesa_fnv32_1a_accumulate_block(hash, key, offsetof(struct key, surf[0]));
	hash = _mesa_fnv32_1a_accumulate_block(hash, key->surf, sizeof(key->surf[0]) * key->num_surfs);
	return hash;
	}

	static bool
	key_equals(const void _a, const void _b)
	{
	const struct key *a = _a;
	const struct key *b = _b;
	return (memcmp(a, b, offsetof(struct key, surf[0])) == 0) &&
	(memcmp(a->surf, b->surf, sizeof(a->surf[0]) * a->num_surfs) == 0);
	}

	void
	fd_bc_init(struct fd_batch_cache *cache)
	{
	cache->ht = _mesa_hash_table_create(NULL, key_hash, key_equals);
	}

	void
	fd_bc_fini(struct fd_batch_cache *cache)
	{
	_mesa_hash_table_destroy(cache->ht, NULL);
	}

	void
	fd_bc_flush(struct fd_batch_cache cache, struct fd_context ctx)
	{
	struct hash_entry *entry;
	struct fd_batch *last_batch = NULL;

	mtx_lock(&ctx->screen->lock);

	hash_table_foreach(cache->ht, entry) {
	struct fd_batch *batch = NULL;
	/* hold a reference since we can drop screen->lock: */
	fd_batch_reference_locked(&batch, (struct fd_batch *)entry->data);
	if (batch->ctx == ctx) {
	mtx_unlock(&ctx->screen->lock);
	fd_batch_reference(&last_batch, batch);
	fd_batch_flush(batch, false, false);
	mtx_lock(&ctx->screen->lock);
	}
	fd_batch_reference_locked(&batch, NULL);
	}

	mtx_unlock(&ctx->screen->lock);

	if (last_batch) {
	fd_batch_sync(last_batch);
	fd_batch_reference(&last_batch, NULL);
	}
	}

	/* deferred flush doesn't actually flush, but it marks every other
	* batch associated with the context as dependent on the current
	* batch. So when the current batch gets flushed, all other batches
	* that came before also get flushed.
	*/
	void
	fd_bc_flush_deferred(struct fd_batch_cache cache, struct fd_context ctx)
	{
	struct fd_batch *current_batch = ctx->batch;
	struct hash_entry *entry;

	mtx_lock(&ctx->screen->lock);

	hash_table_foreach(cache->ht, entry) {
	struct fd_batch *batch = entry->data;
	if (batch == current_batch)
	continue;
	if (batch->ctx == ctx)
	fd_batch_add_dep(current_batch, batch);
	}

	mtx_unlock(&ctx->screen->lock);
	}

	void
	fd_bc_invalidate_context(struct fd_context *ctx)
	{
	struct fd_batch_cache *cache = &ctx->screen->batch_cache;
	struct fd_batch *batch;

	mtx_lock(&ctx->screen->lock);

	foreach_batch(batch, cache, cache->batch_mask) {
	if (batch->ctx == ctx)
	fd_batch_reference_locked(&batch, NULL);
	}

	mtx_unlock(&ctx->screen->lock);
	}

	void
	fd_bc_invalidate_batch(struct fd_batch *batch, bool destroy)
	{
	if (!batch)
	return;

	struct fd_batch_cache *cache = &batch->ctx->screen->batch_cache;
	struct key key = (struct key )batch->key;

	pipe_mutex_assert_locked(batch->ctx->screen->lock);

	if (destroy) {
	cache->batches[batch->idx] = NULL;
	cache->batch_mask &= ~(1 << batch->idx);
	}

	if (!key)
	return;

	DBG("%p: key=%p", batch, batch->key);
	for (unsigned idx = 0; idx < key->num_surfs; idx++) {
	struct fd_resource *rsc = fd_resource(key->surf[idx].texture);
	rsc->bc_batch_mask &= ~(1 << batch->idx);
	}

	struct hash_entry *entry =
	_mesa_hash_table_search_pre_hashed(cache->ht, batch->hash, key);
	_mesa_hash_table_remove(cache->ht, entry);

	batch->key = NULL;
	free(key);
	}

	void
	fd_bc_invalidate_resource(struct fd_resource *rsc, bool destroy)
	{
	struct fd_screen *screen = fd_screen(rsc->base.screen);
	struct fd_batch *batch;

	mtx_lock(&screen->lock);

	if (destroy) {
	foreach_batch(batch, &screen->batch_cache, rsc->batch_mask) {
	struct set_entry *entry = _mesa_set_search(batch->resources, rsc);
	_mesa_set_remove(batch->resources, entry);
	}
	rsc->batch_mask = 0;

	fd_batch_reference_locked(&rsc->write_batch, NULL);
	}

	foreach_batch(batch, &screen->batch_cache, rsc->bc_batch_mask)
	fd_bc_invalidate_batch(batch, false);

	rsc->bc_batch_mask = 0;

	mtx_unlock(&screen->lock);
	}

	struct fd_batch *
	fd_bc_alloc_batch(struct fd_batch_cache cache, struct fd_context ctx)
	{
	struct fd_batch *batch;
	uint32_t idx;

	mtx_lock(&ctx->screen->lock);

	while ((idx = ffs(~cache->batch_mask)) == 0) {
	#if 0
	for (unsigned i = 0; i < ARRAY_SIZE(cache->batches); i++) {
	batch = cache->batches[i];
	debug_printf("%d: needs_flush=%d, depends:", batch->idx, batch->needs_flush);
	struct set_entry *entry;
	set_foreach(batch->dependencies, entry) {
	struct fd_batch dep = (struct fd_batch )entry->key;
	debug_printf(" %d", dep->idx);
	}
	debug_printf("\n");
	}
	#endif
	/* TODO: is LRU the better policy? Or perhaps the batch that
	* depends on the fewest other batches?
	*/
	struct fd_batch *flush_batch = NULL;
	for (unsigned i = 0; i < ARRAY_SIZE(cache->batches); i++) {
	if ((cache->batches[i] == ctx->batch) \|\|
	!cache->batches[i]->needs_flush)
	continue;
	if (!flush_batch \|\| (cache->batches[i]->seqno < flush_batch->seqno))
	fd_batch_reference_locked(&flush_batch, cache->batches[i]);
	}

	/* we can drop lock temporarily here, since we hold a ref,
	* flush_batch won't disappear under us.
	*/
	mtx_unlock(&ctx->screen->lock);
	DBG("%p: too many batches! flush forced!", flush_batch);
	fd_batch_flush(flush_batch, true, false);
	mtx_lock(&ctx->screen->lock);

	/* While the resources get cleaned up automatically, the flush_batch
	* doesn't get removed from the dependencies of other batches, so
	* it won't be unref'd and will remain in the table.
	*
	* TODO maybe keep a bitmask of batches that depend on me, to make
	* this easier:
	*/
	for (unsigned i = 0; i < ARRAY_SIZE(cache->batches); i++) {
	struct fd_batch *other = cache->batches[i];
	if (!other)
	continue;
	if (other->dependents_mask & (1 << flush_batch->idx)) {
	other->dependents_mask &= ~(1 << flush_batch->idx);
	struct fd_batch *ref = flush_batch;
	fd_batch_reference_locked(&ref, NULL);
	}
	}

	fd_batch_reference_locked(&flush_batch, NULL);
	}

	idx--; /* bit zero returns 1 for ffs() */

	batch = fd_batch_create(ctx, false);
	if (!batch)
	goto out;

	batch->seqno = cache->cnt++;
	batch->idx = idx;
	cache->batch_mask \|= (1 << idx);

	debug_assert(cache->batches[idx] == NULL);
	cache->batches[idx] = batch;

	out:
	mtx_unlock(&ctx->screen->lock);

	return batch;
	}

	static struct fd_batch *
	batch_from_key(struct fd_batch_cache cache, struct key key,
	struct fd_context *ctx)
	{
	struct fd_batch *batch = NULL;
	uint32_t hash = key_hash(key);
	struct hash_entry *entry =
	_mesa_hash_table_search_pre_hashed(cache->ht, hash, key);

	if (entry) {
	free(key);
	fd_batch_reference(&batch, (struct fd_batch *)entry->data);
	return batch;
	}

	batch = fd_bc_alloc_batch(cache, ctx);
	#ifdef DEBUG
	DBG("%p: hash=0x%08x, %ux%u, %u layers, %u samples", batch, hash,
	key->width, key->height, key->layers, key->samples);
	for (unsigned idx = 0; idx < key->num_surfs; idx++) {
	DBG("%p: surf[%u]: %p (%s) (%u,%u / %u,%u,%u)", batch, key->surf[idx].pos,
	key->surf[idx].texture, util_format_name(key->surf[idx].format),
	key->surf[idx].u.buf.first_element, key->surf[idx].u.buf.last_element,
	key->surf[idx].u.tex.first_layer, key->surf[idx].u.tex.last_layer,
	key->surf[idx].u.tex.level);
	}
	#endif
	if (!batch)
	return NULL;

	mtx_lock(&ctx->screen->lock);

	_mesa_hash_table_insert_pre_hashed(cache->ht, hash, key, batch);
	batch->key = key;
	batch->hash = hash;

	for (unsigned idx = 0; idx < key->num_surfs; idx++) {
	struct fd_resource *rsc = fd_resource(key->surf[idx].texture);
	rsc->bc_batch_mask = (1 << batch->idx);
	}

	mtx_unlock(&ctx->screen->lock);

	return batch;
	}

	static void
	key_surf(struct key key, unsigned idx, unsigned pos, struct pipe_surface psurf)
	{
	key->surf[idx].texture = psurf->texture;
	key->surf[idx].u = psurf->u;
	key->surf[idx].pos = pos;
	key->surf[idx].format = psurf->format;
	}

	struct fd_batch *
	fd_batch_from_fb(struct fd_batch_cache cache, struct fd_context ctx,
	const struct pipe_framebuffer_state *pfb)
	{
	unsigned idx = 0, n = pfb->nr_cbufs + (pfb->zsbuf ? 1 : 0);
	struct key *key = key_alloc(n);

	key->width = pfb->width;
	key->height = pfb->height;
	key->layers = pfb->layers;
	key->samples = pfb->samples;
	key->ctx = ctx;

	if (pfb->zsbuf)
	key_surf(key, idx++, 0, pfb->zsbuf);

	for (unsigned i = 0; i < pfb->nr_cbufs; i++)
	if (pfb->cbufs[i])
	key_surf(key, idx++, i + 1, pfb->cbufs[i]);

	key->num_surfs = idx;

	return batch_from_key(cache, key, ctx);
	}