drm/i915: Make pin count per VMA

Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>

drivers/gpu/drm/i915/i915_debugfs.c

@@ -100,7 +100,7 @@ static const char *get_pin_flag(struct drm_i915_gem_object *obj)
 {
  if (obj->user_pin_count > 0)
  return "P";
- else if (obj->pin_count > 0)
+ else if (i915_gem_obj_is_pinned(obj))
  return "p";
  else
  return " ";
@@ -125,6 +125,8 @@ static void
 describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
 {
  struct i915_vma *vma;
+ int pin_count = 0;
+
  seq_printf(m, "%pK: %s%s%s %8zdKiB %02x %02x %u %u %u%s%s%s",
  &obj->base,
  get_pin_flag(obj),
@@ -141,8 +143,10 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
  obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
  if (obj->base.name)
  seq_printf(m, " (name: %d)", obj->base.name);
- if (obj->pin_count)
- seq_printf(m, " (pinned x %d)", obj->pin_count);
+ list_for_each_entry(vma, &obj->vma_list, vma_link)
+ if (vma->pin_count > 0)
+ pin_count++;
+ seq_printf(m, " (pinned x %d)", pin_count);
  if (obj->pin_display)
  seq_printf(m, " (display)");
  if (obj->fence_reg != I915_FENCE_REG_NONE)
@@ -439,7 +443,7 @@ static int i915_gem_gtt_info(struct seq_file *m, void *data)
 
  total_obj_size = total_gtt_size = count = 0;
  list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
- if (list == PINNED_LIST && obj->pin_count == 0)
+ if (list == PINNED_LIST && !i915_gem_obj_is_pinned(obj))
  continue;
 
  seq_puts(m, " ");
@@ -2843,7 +2847,7 @@ i915_drop_caches_set(void *data, u64 val)
  list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
  list_for_each_entry_safe(vma, x, &vm->inactive_list,
  mm_list) {
- if (vma->obj->pin_count)
+ if (vma->pin_count)
  continue;
 
  ret = i915_vma_unbind(vma);

drivers/gpu/drm/i915/i915_drv.h

@@ -651,6 +651,19 @@ struct i915_vma {
  unsigned long exec_handle;
  struct drm_i915_gem_exec_object2 *exec_entry;
 
+ /**
+ * How many users have pinned this object in GTT space. The following
+ * users can each hold at most one reference: pwrite/pread, pin_ioctl
+ * (via user_pin_count), execbuffer (objects are not allowed multiple
+ * times for the same batchbuffer), and the framebuffer code. When
+ * switching/pageflipping, the framebuffer code has at most two buffers
+ * pinned per crtc.
+ *
+ * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3
+ * bits with absolutely no headroom. So use 4 bits.
+ */
+ unsigned int pin_count:4;
+#define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf
 };
 
 struct i915_ctx_hang_stats {
@@ -1617,18 +1630,6 @@ struct drm_i915_gem_object {
  */
  unsigned int fence_dirty:1;
 
- /** How many users have pinned this object in GTT space. The following
- * users can each hold at most one reference: pwrite/pread, pin_ioctl
- * (via user_pin_count), execbuffer (objects are not allowed multiple
- * times for the same batchbuffer), and the framebuffer code. When
- * switching/pageflipping, the framebuffer code has at most two buffers
- * pinned per crtc.
- *
- * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3
- * bits with absolutely no headroom. So use 4 bits. */
- unsigned int pin_count:4;
-#define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf
-
  /**
  * Is the object at the current location in the gtt mappable and
  * fenceable? Used to avoid costly recalculations.
@@ -2005,7 +2006,7 @@ int __must_check i915_gem_object_pin(struct drm_i915_gem_object *obj,
  uint32_t alignment,
  bool map_and_fenceable,
  bool nonblocking);
-void i915_gem_object_unpin(struct drm_i915_gem_object *obj);
+void i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj);
 int __must_check i915_vma_unbind(struct i915_vma *vma);
 int __must_check i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj);
 int i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
@@ -2168,6 +2169,13 @@ i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
  struct i915_address_space *vm);
 
 struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj);
+static inline bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj) {
+ struct i915_vma *vma;
+ list_for_each_entry(vma, &obj->vma_list, vma_link)
+ if (vma->pin_count > 0)
+ return true;
+ return false;
+}
 
 /* Some GGTT VM helpers */
 #define obj_to_ggtt(obj) \

drivers/gpu/drm/i915/i915_gem.c

@@ -204,7 +204,7 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  pinned = 0;
  mutex_lock(&dev->struct_mutex);
  list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
- if (obj->pin_count)
+ if (i915_gem_obj_is_pinned(obj))
  pinned += i915_gem_obj_ggtt_size(obj);
  mutex_unlock(&dev->struct_mutex);
 
@@ -651,7 +651,7 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev,
  }
 
 out_unpin:
- i915_gem_object_unpin(obj);
+ i915_gem_object_ggtt_unpin(obj);
 out:
  return ret;
 }
@@ -1418,7 +1418,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  /* Finally, remap it using the new GTT offset */
  ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
 unpin:
- i915_gem_object_unpin(obj);
+ i915_gem_object_ggtt_unpin(obj);
 unlock:
  mutex_unlock(&dev->struct_mutex);
 out:
@@ -2721,7 +2721,7 @@ int i915_vma_unbind(struct i915_vma *vma)
  return 0;
  }
 
- if (obj->pin_count)
+ if (vma->pin_count)
  return -EBUSY;
 
  BUG_ON(obj->pages == NULL);
@@ -2785,7 +2785,7 @@ i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj)
  if (!i915_gem_obj_ggtt_bound(obj))
  return 0;
 
- if (obj->pin_count)
+ if (i915_gem_obj_to_ggtt(obj)->pin_count)
  return -EBUSY;
 
  BUG_ON(obj->pages == NULL);
@@ -3486,7 +3486,7 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
  if (obj->cache_level == cache_level)
  return 0;
 
- if (obj->pin_count) {
+ if (i915_gem_obj_is_pinned(obj)) {
  DRM_DEBUG("can not change the cache level of pinned objects\n");
  return -EBUSY;
  }
@@ -3646,7 +3646,7 @@ static bool is_pin_display(struct drm_i915_gem_object *obj)
  * subtracting the potential reference by the user, any pin_count
  * remains, it must be due to another use by the display engine.
  */
- return obj->pin_count - !!obj->user_pin_count;
+ return i915_gem_obj_to_ggtt(obj)->pin_count - !!obj->user_pin_count;
 }
 
 /*
@@ -3720,7 +3720,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 void
 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
 {
- i915_gem_object_unpin(obj);
+ i915_gem_object_ggtt_unpin(obj);
  obj->pin_display = is_pin_display(obj);
 }
 
@@ -3853,18 +3853,18 @@ i915_gem_object_pin(struct drm_i915_gem_object *obj,
  struct i915_vma *vma;
  int ret;
 
- if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
- return -EBUSY;
-
  WARN_ON(map_and_fenceable && !i915_is_ggtt(vm));
 
  vma = i915_gem_obj_to_vma(obj, vm);
 
  if (vma) {
+ if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
+ return -EBUSY;
+
  if ((alignment &&
  vma->node.start & (alignment - 1)) ||
  (map_and_fenceable && !obj->map_and_fenceable)) {
- WARN(obj->pin_count,
+ WARN(vma->pin_count,
  "bo is already pinned with incorrect alignment:"
  " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
  " obj->map_and_fenceable=%d\n",
@@ -3893,19 +3893,22 @@ i915_gem_object_pin(struct drm_i915_gem_object *obj,
  if (!obj->has_global_gtt_mapping && map_and_fenceable)
  i915_gem_gtt_bind_object(obj, obj->cache_level);
 
- obj->pin_count++;
+ i915_gem_obj_to_vma(obj, vm)->pin_count++;
  obj->pin_mappable |= map_and_fenceable;
 
  return 0;
 }
 
 void
-i915_gem_object_unpin(struct drm_i915_gem_object *obj)
+i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj)
 {
- BUG_ON(obj->pin_count == 0);
- BUG_ON(!i915_gem_obj_bound_any(obj));
+ struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
 
- if (--obj->pin_count == 0)
+ BUG_ON(!vma);
+ BUG_ON(vma->pin_count == 0);
+ BUG_ON(!i915_gem_obj_ggtt_bound(obj));
+
+ if (--vma->pin_count == 0)
  obj->pin_mappable = false;
 }
 
@@ -3989,7 +3992,7 @@ i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
  obj->user_pin_count--;
  if (obj->user_pin_count == 0) {
  obj->pin_filp = NULL;
- i915_gem_object_unpin(obj);
+ i915_gem_object_ggtt_unpin(obj);
  }
 
 out:
@@ -4069,7 +4072,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  goto unlock;
  }
 
- if (obj->pin_count) {
+ if (i915_gem_obj_is_pinned(obj)) {
  ret = -EINVAL;
  goto out;
  }
@@ -4178,12 +4181,14 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
  if (obj->phys_obj)
  i915_gem_detach_phys_object(dev, obj);
 
- obj->pin_count = 0;
  /* NB: 0 or 1 elements */
  WARN_ON(!list_empty(&obj->vma_list) &&
  !list_is_singular(&obj->vma_list));
  list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
- int ret = i915_vma_unbind(vma);
+ int ret;
+
+ vma->pin_count = 0;
+ ret = i915_vma_unbind(vma);
  if (WARN_ON(ret == -ERESTARTSYS)) {
  bool was_interruptible;
 
@@ -4963,7 +4968,7 @@ i915_gem_inactive_count(struct shrinker *shrinker, struct shrink_control *sc)
  if (obj->active)
  continue;
 
- if (obj->pin_count == 0 && obj->pages_pin_count == 0)
+ if (!i915_gem_obj_is_pinned(obj) && obj->pages_pin_count == 0)
  count += obj->base.size >> PAGE_SHIFT;
  }
 

drivers/gpu/drm/i915/i915_gem_context.c

@@ -241,7 +241,7 @@ static int create_default_context(struct drm_i915_private *dev_priv)
  return 0;
 
 err_unpin:
- i915_gem_object_unpin(ctx->obj);
+ i915_gem_object_ggtt_unpin(ctx->obj);
 err_destroy:
  i915_gem_context_unreference(ctx);
  return ret;
@@ -300,11 +300,11 @@ void i915_gem_context_fini(struct drm_device *dev)
  if (dev_priv->ring[RCS].last_context == dctx) {
  /* Fake switch to NULL context */
  WARN_ON(dctx->obj->active);
- i915_gem_object_unpin(dctx->obj);
+ i915_gem_object_ggtt_unpin(dctx->obj);
  i915_gem_context_unreference(dctx);
  }
 
- i915_gem_object_unpin(dctx->obj);
+ i915_gem_object_ggtt_unpin(dctx->obj);
  i915_gem_context_unreference(dctx);
  dev_priv->ring[RCS].default_context = NULL;
  dev_priv->ring[RCS].last_context = NULL;
@@ -412,7 +412,7 @@ static int do_switch(struct i915_hw_context *to)
  u32 hw_flags = 0;
  int ret, i;
 
- BUG_ON(from != NULL && from->obj != NULL && from->obj->pin_count == 0);
+ BUG_ON(from != NULL && from->obj != NULL && !i915_gem_obj_is_pinned(from->obj));
 
  if (from == to && !to->remap_slice)
  return 0;
@@ -428,7 +428,7 @@ static int do_switch(struct i915_hw_context *to)
  * XXX: We need a real interface to do this instead of trickery. */
  ret = i915_gem_object_set_to_gtt_domain(to->obj, false);
  if (ret) {
- i915_gem_object_unpin(to->obj);
+ i915_gem_object_ggtt_unpin(to->obj);
  return ret;
  }
 
@@ -440,7 +440,7 @@ static int do_switch(struct i915_hw_context *to)
 
  ret = mi_set_context(ring, to, hw_flags);
  if (ret) {
- i915_gem_object_unpin(to->obj);
+ i915_gem_object_ggtt_unpin(to->obj);
  return ret;
  }
 
@@ -476,7 +476,7 @@ static int do_switch(struct i915_hw_context *to)
  BUG_ON(from->obj->ring != ring);
 
  /* obj is kept alive until the next request by its active ref */
- i915_gem_object_unpin(from->obj);
+ i915_gem_object_ggtt_unpin(from->obj);
  i915_gem_context_unreference(from);
  }
 

drivers/gpu/drm/i915/i915_gem_evict.c

@@ -34,7 +34,8 @@
 static bool
 mark_free(struct i915_vma *vma, struct list_head *unwind)
 {
- if (vma->obj->pin_count)
+ /* Freeing up memory requires no VMAs are pinned */
+ if (i915_gem_obj_is_pinned(vma->obj))
  return false;
 
  if (WARN_ON(!list_empty(&vma->exec_list)))
@@ -186,7 +187,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)
  }
 
  list_for_each_entry_safe(vma, next, &vm->inactive_list, mm_list)
- if (vma->obj->pin_count == 0)
+ if (vma->pin_count == 0)
  WARN_ON(i915_vma_unbind(vma));
 
  return 0;

drivers/gpu/drm/i915/i915_gem_execbuffer.c

@@ -566,7 +566,7 @@ i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
  i915_gem_object_unpin_fence(obj);
 
  if (entry->flags & __EXEC_OBJECT_HAS_PIN)
- i915_gem_object_unpin(obj);
+ vma->pin_count--;
 
  entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
 }
@@ -923,7 +923,9 @@ i915_gem_execbuffer_move_to_active(struct list_head *vmas,
  if (obj->base.write_domain) {
  obj->dirty = 1;
  obj->last_write_seqno = intel_ring_get_seqno(ring);
- if (obj->pin_count) /* check for potential scanout */
+ /* check for potential scanout */
+ if (i915_gem_obj_ggtt_bound(obj) &&
+ i915_gem_obj_to_ggtt(obj)->pin_count)
  intel_mark_fb_busy(obj, ring);
  }
 

drivers/gpu/drm/i915/i915_gem_tiling.c

@@ -308,7 +308,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
  return -EINVAL;
  }
 
- if (obj->pin_count || obj->framebuffer_references) {
+ if (i915_gem_obj_is_pinned(obj) || obj->framebuffer_references) {
  drm_gem_object_unreference_unlocked(&obj->base);
  return -EBUSY;
  }