drivers/gpu/drm/vmwgfx/vmwgfx_fence.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /**************************************************************************
   3  *
   4  * Copyright 2011-2023 VMware, Inc., Palo Alto, CA., USA
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include <linux/sched/signal.h>
  29
  30 #include "vmwgfx_drv.h"
  31
  32 #define VMW_FENCE_WRAP (1 << 31)
  33
  34 struct vmw_fence_manager {
  35         int num_fence_objects;
  36         struct vmw_private *dev_priv;
  37         spinlock_t lock;
  38         struct list_head fence_list;
  39         struct work_struct work;
  40         bool fifo_down;
  41         struct list_head cleanup_list;
  42         uint32_t pending_actions[VMW_ACTION_MAX];
  43         struct mutex goal_irq_mutex;
  44         bool goal_irq_on; /* Protected by @goal_irq_mutex */
  45         bool seqno_valid; /* Protected by @lock, and may not be set to true
  46                              without the @goal_irq_mutex held. */
  47         u64 ctx;
  48 };
  49
  50 struct vmw_user_fence {
  51         struct ttm_base_object base;
  52         struct vmw_fence_obj fence;
  53 };
  54
  55 /**
  56  * struct vmw_event_fence_action - fence action that delivers a drm event.
  57  *
  58  * @action: A struct vmw_fence_action to hook up to a fence.
  59  * @event: A pointer to the pending event.
  60  * @fence: A referenced pointer to the fence to keep it alive while @action
  61  * hangs on it.
  62  * @dev: Pointer to a struct drm_device so we can access the event stuff.
  63  * @tv_sec: If non-null, the variable pointed to will be assigned
  64  * current time tv_sec val when the fence signals.
  65  * @tv_usec: Must be set if @tv_sec is set, and the variable pointed to will
  66  * be assigned the current time tv_usec val when the fence signals.
  67  */
  68 struct vmw_event_fence_action {
  69         struct vmw_fence_action action;
  70
  71         struct drm_pending_event *event;
  72         struct vmw_fence_obj *fence;
  73         struct drm_device *dev;
  74
  75         uint32_t *tv_sec;
  76         uint32_t *tv_usec;
  77 };
  78
  79 static struct vmw_fence_manager *
  80 fman_from_fence(struct vmw_fence_obj *fence)
  81 {
  82         return container_of(fence->base.lock, struct vmw_fence_manager, lock);
  83 }
  84
  85 static u32 vmw_fence_goal_read(struct vmw_private *vmw)
  86 {
  87         if ((vmw->capabilities2 & SVGA_CAP2_EXTRA_REGS) != 0)
  88                 return vmw_read(vmw, SVGA_REG_FENCE_GOAL);
  89         else
  90                 return vmw_fifo_mem_read(vmw, SVGA_FIFO_FENCE_GOAL);
  91 }
  92
  93 static void vmw_fence_goal_write(struct vmw_private *vmw, u32 value)
  94 {
  95         if ((vmw->capabilities2 & SVGA_CAP2_EXTRA_REGS) != 0)
  96                 vmw_write(vmw, SVGA_REG_FENCE_GOAL, value);
  97         else
  98                 vmw_fifo_mem_write(vmw, SVGA_FIFO_FENCE_GOAL, value);
  99 }
 100
 101 /*
 102  * Note on fencing subsystem usage of irqs:
 103  * Typically the vmw_fences_update function is called
 104  *
 105  * a) When a new fence seqno has been submitted by the fifo code.
 106  * b) On-demand when we have waiters. Sleeping waiters will switch on the
 107  * ANY_FENCE irq and call vmw_fences_update function each time an ANY_FENCE
 108  * irq is received. When the last fence waiter is gone, that IRQ is masked
 109  * away.
 110  *
 111  * In situations where there are no waiters and we don't submit any new fences,
 112  * fence objects may not be signaled. This is perfectly OK, since there are
 113  * no consumers of the signaled data, but that is NOT ok when there are fence
 114  * actions attached to a fence. The fencing subsystem then makes use of the
 115  * FENCE_GOAL irq and sets the fence goal seqno to that of the next fence
 116  * which has an action attached, and each time vmw_fences_update is called,
 117  * the subsystem makes sure the fence goal seqno is updated.
 118  *
 119  * The fence goal seqno irq is on as long as there are unsignaled fence
 120  * objects with actions attached to them.
 121  */
 122
 123 static void vmw_fence_obj_destroy(struct dma_fence *f)
 124 {
 125         struct vmw_fence_obj *fence =
 126                 container_of(f, struct vmw_fence_obj, base);
 127
 128         struct vmw_fence_manager *fman = fman_from_fence(fence);
 129
 130         spin_lock(&fman->lock);
 131         list_del_init(&fence->head);
 132         --fman->num_fence_objects;
 133         spin_unlock(&fman->lock);
 134         fence->destroy(fence);
 135 }
 136
 137 static const char *vmw_fence_get_driver_name(struct dma_fence *f)
 138 {
 139         return "vmwgfx";
 140 }
 141
 142 static const char *vmw_fence_get_timeline_name(struct dma_fence *f)
 143 {
 144         return "svga";
 145 }
 146
 147 static bool vmw_fence_enable_signaling(struct dma_fence *f)
 148 {
 149         struct vmw_fence_obj *fence =
 150                 container_of(f, struct vmw_fence_obj, base);
 151
 152         struct vmw_fence_manager *fman = fman_from_fence(fence);
 153         struct vmw_private *dev_priv = fman->dev_priv;
 154
 155         u32 seqno = vmw_fence_read(dev_priv);
 156         if (seqno - fence->base.seqno < VMW_FENCE_WRAP)
 157                 return false;
 158
 159         return true;
 160 }
 161
 162 struct vmwgfx_wait_cb {
 163         struct dma_fence_cb base;
 164         struct task_struct *task;
 165 };
 166
 167 static void
 168 vmwgfx_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
 169 {
 170         struct vmwgfx_wait_cb *wait =
 171                 container_of(cb, struct vmwgfx_wait_cb, base);
 172
 173         wake_up_process(wait->task);
 174 }
 175
 176 static void __vmw_fences_update(struct vmw_fence_manager *fman);
 177
 178 static long vmw_fence_wait(struct dma_fence *f, bool intr, signed long timeout)
 179 {
 180         struct vmw_fence_obj *fence =
 181                 container_of(f, struct vmw_fence_obj, base);
 182
 183         struct vmw_fence_manager *fman = fman_from_fence(fence);
 184         struct vmw_private *dev_priv = fman->dev_priv;
 185         struct vmwgfx_wait_cb cb;
 186         long ret = timeout;
 187
 188         if (likely(vmw_fence_obj_signaled(fence)))
 189                 return timeout;
 190
 191         vmw_seqno_waiter_add(dev_priv);
 192
 193         spin_lock(f->lock);
 194
 195         if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->flags))
 196                 goto out;
 197
 198         if (intr && signal_pending(current)) {
 199                 ret = -ERESTARTSYS;
 200                 goto out;
 201         }
 202
 203         cb.base.func = vmwgfx_wait_cb;
 204         cb.task = current;
 205         list_add(&cb.base.node, &f->cb_list);
 206
 207         for (;;) {
 208                 __vmw_fences_update(fman);
 209
 210                 /*
 211                  * We can use the barrier free __set_current_state() since
 212                  * DMA_FENCE_FLAG_SIGNALED_BIT + wakeup is protected by the
 213                  * fence spinlock.
 214                  */
 215                 if (intr)
 216                         __set_current_state(TASK_INTERRUPTIBLE);
 217                 else
 218                         __set_current_state(TASK_UNINTERRUPTIBLE);
 219
 220                 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->flags)) {
 221                         if (ret == 0 && timeout > 0)
 222                                 ret = 1;
 223                         break;
 224                 }
 225
 226                 if (intr && signal_pending(current)) {
 227                         ret = -ERESTARTSYS;
 228                         break;
 229                 }
 230
 231                 if (ret == 0)
 232                         break;
 233
 234                 spin_unlock(f->lock);
 235
 236                 ret = schedule_timeout(ret);
 237
 238                 spin_lock(f->lock);
 239         }
 240         __set_current_state(TASK_RUNNING);
 241         if (!list_empty(&cb.base.node))
 242                 list_del(&cb.base.node);
 243
 244 out:
 245         spin_unlock(f->lock);
 246
 247         vmw_seqno_waiter_remove(dev_priv);
 248
 249         return ret;
 250 }
 251
 252 static const struct dma_fence_ops vmw_fence_ops = {
 253         .get_driver_name = vmw_fence_get_driver_name,
 254         .get_timeline_name = vmw_fence_get_timeline_name,
 255         .enable_signaling = vmw_fence_enable_signaling,
 256         .wait = vmw_fence_wait,
 257         .release = vmw_fence_obj_destroy,
 258 };
 259
 260
 261 /*
 262  * Execute signal actions on fences recently signaled.
 263  * This is done from a workqueue so we don't have to execute
 264  * signal actions from atomic context.
 265  */
 266
 267 static void vmw_fence_work_func(struct work_struct *work)
 268 {
 269         struct vmw_fence_manager *fman =
 270                 container_of(work, struct vmw_fence_manager, work);
 271         struct list_head list;
 272         struct vmw_fence_action *action, *next_action;
 273         bool seqno_valid;
 274
 275         do {
 276                 INIT_LIST_HEAD(&list);
 277                 mutex_lock(&fman->goal_irq_mutex);
 278
 279                 spin_lock(&fman->lock);
 280                 list_splice_init(&fman->cleanup_list, &list);
 281                 seqno_valid = fman->seqno_valid;
 282                 spin_unlock(&fman->lock);
 283
 284                 if (!seqno_valid && fman->goal_irq_on) {
 285                         fman->goal_irq_on = false;
 286                         vmw_goal_waiter_remove(fman->dev_priv);
 287                 }
 288                 mutex_unlock(&fman->goal_irq_mutex);
 289
 290                 if (list_empty(&list))
 291                         return;
 292
 293                 /*
 294                  * At this point, only we should be able to manipulate the
 295                  * list heads of the actions we have on the private list.
 296                  * hence fman::lock not held.
 297                  */
 298
 299                 list_for_each_entry_safe(action, next_action, &list, head) {
 300                         list_del_init(&action->head);
 301                         if (action->cleanup)
 302                                 action->cleanup(action);
 303                 }
 304         } while (1);
 305 }
 306
 307 struct vmw_fence_manager *vmw_fence_manager_init(struct vmw_private *dev_priv)
 308 {
 309         struct vmw_fence_manager *fman = kzalloc(sizeof(*fman), GFP_KERNEL);
 310
 311         if (unlikely(!fman))
 312                 return NULL;
 313
 314         fman->dev_priv = dev_priv;
 315         spin_lock_init(&fman->lock);
 316         INIT_LIST_HEAD(&fman->fence_list);
 317         INIT_LIST_HEAD(&fman->cleanup_list);
 318         INIT_WORK(&fman->work, &vmw_fence_work_func);
 319         fman->fifo_down = true;
 320         mutex_init(&fman->goal_irq_mutex);
 321         fman->ctx = dma_fence_context_alloc(1);
 322
 323         return fman;
 324 }
 325
 326 void vmw_fence_manager_takedown(struct vmw_fence_manager *fman)
 327 {
 328         bool lists_empty;
 329
 330         (void) cancel_work_sync(&fman->work);
 331
 332         spin_lock(&fman->lock);
 333         lists_empty = list_empty(&fman->fence_list) &&
 334                 list_empty(&fman->cleanup_list);
 335         spin_unlock(&fman->lock);
 336
 337         BUG_ON(!lists_empty);
 338         kfree(fman);
 339 }
 340
 341 static int vmw_fence_obj_init(struct vmw_fence_manager *fman,
 342                               struct vmw_fence_obj *fence, u32 seqno,
 343                               void (*destroy) (struct vmw_fence_obj *fence))
 344 {
 345         int ret = 0;
 346
 347         dma_fence_init(&fence->base, &vmw_fence_ops, &fman->lock,
 348                        fman->ctx, seqno);
 349         INIT_LIST_HEAD(&fence->seq_passed_actions);
 350         fence->destroy = destroy;
 351
 352         spin_lock(&fman->lock);
 353         if (unlikely(fman->fifo_down)) {
 354                 ret = -EBUSY;
 355                 goto out_unlock;
 356         }
 357         list_add_tail(&fence->head, &fman->fence_list);
 358         ++fman->num_fence_objects;
 359
 360 out_unlock:
 361         spin_unlock(&fman->lock);
 362         return ret;
 363
 364 }
 365
 366 static void vmw_fences_perform_actions(struct vmw_fence_manager *fman,
 367                                 struct list_head *list)
 368 {
 369         struct vmw_fence_action *action, *next_action;
 370
 371         list_for_each_entry_safe(action, next_action, list, head) {
 372                 list_del_init(&action->head);
 373                 fman->pending_actions[action->type]--;
 374                 if (action->seq_passed != NULL)
 375                         action->seq_passed(action);
 376
 377                 /*
 378                  * Add the cleanup action to the cleanup list so that
 379                  * it will be performed by a worker task.
 380                  */
 381
 382                 list_add_tail(&action->head, &fman->cleanup_list);
 383         }
 384 }
 385
 386 /**
 387  * vmw_fence_goal_new_locked - Figure out a new device fence goal
 388  * seqno if needed.
 389  *
 390  * @fman: Pointer to a fence manager.
 391  * @passed_seqno: The seqno the device currently signals as passed.
 392  *
 393  * This function should be called with the fence manager lock held.
 394  * It is typically called when we have a new passed_seqno, and
 395  * we might need to update the fence goal. It checks to see whether
 396  * the current fence goal has already passed, and, in that case,
 397  * scans through all unsignaled fences to get the next fence object with an
 398  * action attached, and sets the seqno of that fence as a new fence goal.
 399  *
 400  * returns true if the device goal seqno was updated. False otherwise.
 401  */
 402 static bool vmw_fence_goal_new_locked(struct vmw_fence_manager *fman,
 403                                       u32 passed_seqno)
 404 {
 405         u32 goal_seqno;
 406         struct vmw_fence_obj *fence;
 407
 408         if (likely(!fman->seqno_valid))
 409                 return false;
 410
 411         goal_seqno = vmw_fence_goal_read(fman->dev_priv);
 412         if (likely(passed_seqno - goal_seqno >= VMW_FENCE_WRAP))
 413                 return false;
 414
 415         fman->seqno_valid = false;
 416         list_for_each_entry(fence, &fman->fence_list, head) {
 417                 if (!list_empty(&fence->seq_passed_actions)) {
 418                         fman->seqno_valid = true;
 419                         vmw_fence_goal_write(fman->dev_priv,
 420                                              fence->base.seqno);
 421                         break;
 422                 }
 423         }
 424
 425         return true;
 426 }
 427
 428
 429 /**
 430  * vmw_fence_goal_check_locked - Replace the device fence goal seqno if
 431  * needed.
 432  *
 433  * @fence: Pointer to a struct vmw_fence_obj the seqno of which should be
 434  * considered as a device fence goal.
 435  *
 436  * This function should be called with the fence manager lock held.
 437  * It is typically called when an action has been attached to a fence to
 438  * check whether the seqno of that fence should be used for a fence
 439  * goal interrupt. This is typically needed if the current fence goal is
 440  * invalid, or has a higher seqno than that of the current fence object.
 441  *
 442  * returns true if the device goal seqno was updated. False otherwise.
 443  */
 444 static bool vmw_fence_goal_check_locked(struct vmw_fence_obj *fence)
 445 {
 446         struct vmw_fence_manager *fman = fman_from_fence(fence);
 447         u32 goal_seqno;
 448
 449         if (dma_fence_is_signaled_locked(&fence->base))
 450                 return false;
 451
 452         goal_seqno = vmw_fence_goal_read(fman->dev_priv);
 453         if (likely(fman->seqno_valid &&
 454                    goal_seqno - fence->base.seqno < VMW_FENCE_WRAP))
 455                 return false;
 456
 457         vmw_fence_goal_write(fman->dev_priv, fence->base.seqno);
 458         fman->seqno_valid = true;
 459
 460         return true;
 461 }
 462
 463 static void __vmw_fences_update(struct vmw_fence_manager *fman)
 464 {
 465         struct vmw_fence_obj *fence, *next_fence;
 466         struct list_head action_list;
 467         bool needs_rerun;
 468         uint32_t seqno, new_seqno;
 469
 470         seqno = vmw_fence_read(fman->dev_priv);
 471 rerun:
 472         list_for_each_entry_safe(fence, next_fence, &fman->fence_list, head) {
 473                 if (seqno - fence->base.seqno < VMW_FENCE_WRAP) {
 474                         list_del_init(&fence->head);
 475                         dma_fence_signal_locked(&fence->base);
 476                         INIT_LIST_HEAD(&action_list);
 477                         list_splice_init(&fence->seq_passed_actions,
 478                                          &action_list);
 479                         vmw_fences_perform_actions(fman, &action_list);
 480                 } else
 481                         break;
 482         }
 483
 484         /*
 485          * Rerun if the fence goal seqno was updated, and the
 486          * hardware might have raced with that update, so that
 487          * we missed a fence_goal irq.
 488          */
 489
 490         needs_rerun = vmw_fence_goal_new_locked(fman, seqno);
 491         if (unlikely(needs_rerun)) {
 492                 new_seqno = vmw_fence_read(fman->dev_priv);
 493                 if (new_seqno != seqno) {
 494                         seqno = new_seqno;
 495                         goto rerun;
 496                 }
 497         }
 498
 499         if (!list_empty(&fman->cleanup_list))
 500                 (void) schedule_work(&fman->work);
 501 }
 502
 503 void vmw_fences_update(struct vmw_fence_manager *fman)
 504 {
 505         spin_lock(&fman->lock);
 506         __vmw_fences_update(fman);
 507         spin_unlock(&fman->lock);
 508 }
 509
 510 bool vmw_fence_obj_signaled(struct vmw_fence_obj *fence)
 511 {
 512         struct vmw_fence_manager *fman = fman_from_fence(fence);
 513
 514         if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
 515                 return true;
 516
 517         vmw_fences_update(fman);
 518
 519         return dma_fence_is_signaled(&fence->base);
 520 }
 521
 522 int vmw_fence_obj_wait(struct vmw_fence_obj *fence, bool lazy,
 523                        bool interruptible, unsigned long timeout)
 524 {
 525         long ret = dma_fence_wait_timeout(&fence->base, interruptible, timeout);
 526
 527         if (likely(ret > 0))
 528                 return 0;
 529         else if (ret == 0)
 530                 return -EBUSY;
 531         else
 532                 return ret;
 533 }
 534
 535 static void vmw_fence_destroy(struct vmw_fence_obj *fence)
 536 {
 537         dma_fence_free(&fence->base);
 538 }
 539
 540 int vmw_fence_create(struct vmw_fence_manager *fman,
 541                      uint32_t seqno,
 542                      struct vmw_fence_obj **p_fence)
 543 {
 544         struct vmw_fence_obj *fence;
 545         int ret;
 546
 547         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 548         if (unlikely(!fence))
 549                 return -ENOMEM;
 550
 551         ret = vmw_fence_obj_init(fman, fence, seqno,
 552                                  vmw_fence_destroy);
 553         if (unlikely(ret != 0))
 554                 goto out_err_init;
 555
 556         *p_fence = fence;
 557         return 0;
 558
 559 out_err_init:
 560         kfree(fence);
 561         return ret;
 562 }
 563
 564
 565 static void vmw_user_fence_destroy(struct vmw_fence_obj *fence)
 566 {
 567         struct vmw_user_fence *ufence =
 568                 container_of(fence, struct vmw_user_fence, fence);
 569
 570         ttm_base_object_kfree(ufence, base);
 571 }
 572
 573 static void vmw_user_fence_base_release(struct ttm_base_object **p_base)
 574 {
 575         struct ttm_base_object *base = *p_base;
 576         struct vmw_user_fence *ufence =
 577                 container_of(base, struct vmw_user_fence, base);
 578         struct vmw_fence_obj *fence = &ufence->fence;
 579
 580         *p_base = NULL;
 581         vmw_fence_obj_unreference(&fence);
 582 }
 583
 584 int vmw_user_fence_create(struct drm_file *file_priv,
 585                           struct vmw_fence_manager *fman,
 586                           uint32_t seqno,
 587                           struct vmw_fence_obj **p_fence,
 588                           uint32_t *p_handle)
 589 {
 590         struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
 591         struct vmw_user_fence *ufence;
 592         struct vmw_fence_obj *tmp;
 593         int ret;
 594
 595         ufence = kzalloc(sizeof(*ufence), GFP_KERNEL);
 596         if (unlikely(!ufence)) {
 597                 ret = -ENOMEM;
 598                 goto out_no_object;
 599         }
 600
 601         ret = vmw_fence_obj_init(fman, &ufence->fence, seqno,
 602                                  vmw_user_fence_destroy);
 603         if (unlikely(ret != 0)) {
 604                 kfree(ufence);
 605                 goto out_no_object;
 606         }
 607
 608         /*
 609          * The base object holds a reference which is freed in
 610          * vmw_user_fence_base_release.
 611          */
 612         tmp = vmw_fence_obj_reference(&ufence->fence);
 613
 614         ret = ttm_base_object_init(tfile, &ufence->base, false,
 615                                    VMW_RES_FENCE,
 616                                    &vmw_user_fence_base_release);
 617
 618
 619         if (unlikely(ret != 0)) {
 620                 /*
 621                  * Free the base object's reference
 622                  */
 623                 vmw_fence_obj_unreference(&tmp);
 624                 goto out_err;
 625         }
 626
 627         *p_fence = &ufence->fence;
 628         *p_handle = ufence->base.handle;
 629
 630         return 0;
 631 out_err:
 632         tmp = &ufence->fence;
 633         vmw_fence_obj_unreference(&tmp);
 634 out_no_object:
 635         return ret;
 636 }
 637
 638 /*
 639  * vmw_fence_fifo_down - signal all unsignaled fence objects.
 640  */
 641
 642 void vmw_fence_fifo_down(struct vmw_fence_manager *fman)
 643 {
 644         struct list_head action_list;
 645         int ret;
 646
 647         /*
 648          * The list may be altered while we traverse it, so always
 649          * restart when we've released the fman->lock.
 650          */
 651
 652         spin_lock(&fman->lock);
 653         fman->fifo_down = true;
 654         while (!list_empty(&fman->fence_list)) {
 655                 struct vmw_fence_obj *fence =
 656                         list_entry(fman->fence_list.prev, struct vmw_fence_obj,
 657                                    head);
 658                 dma_fence_get(&fence->base);
 659                 spin_unlock(&fman->lock);
 660
 661                 ret = vmw_fence_obj_wait(fence, false, false,
 662                                          VMW_FENCE_WAIT_TIMEOUT);
 663
 664                 if (unlikely(ret != 0)) {
 665                         list_del_init(&fence->head);
 666                         dma_fence_signal(&fence->base);
 667                         INIT_LIST_HEAD(&action_list);
 668                         list_splice_init(&fence->seq_passed_actions,
 669                                          &action_list);
 670                         vmw_fences_perform_actions(fman, &action_list);
 671                 }
 672
 673                 BUG_ON(!list_empty(&fence->head));
 674                 dma_fence_put(&fence->base);
 675                 spin_lock(&fman->lock);
 676         }
 677         spin_unlock(&fman->lock);
 678 }
 679
 680 void vmw_fence_fifo_up(struct vmw_fence_manager *fman)
 681 {
 682         spin_lock(&fman->lock);
 683         fman->fifo_down = false;
 684         spin_unlock(&fman->lock);
 685 }
 686
 687
 688 /**
 689  * vmw_fence_obj_lookup - Look up a user-space fence object
 690  *
 691  * @tfile: A struct ttm_object_file identifying the caller.
 692  * @handle: A handle identifying the fence object.
 693  * @return: A struct vmw_user_fence base ttm object on success or
 694  * an error pointer on failure.
 695  *
 696  * The fence object is looked up and type-checked. The caller needs
 697  * to have opened the fence object first, but since that happens on
 698  * creation and fence objects aren't shareable, that's not an
 699  * issue currently.
 700  */
 701 static struct ttm_base_object *
 702 vmw_fence_obj_lookup(struct ttm_object_file *tfile, u32 handle)
 703 {
 704         struct ttm_base_object *base = ttm_base_object_lookup(tfile, handle);
 705
 706         if (!base) {
 707                 pr_err("Invalid fence object handle 0x%08lx.\n",
 708                        (unsigned long)handle);
 709                 return ERR_PTR(-EINVAL);
 710         }
 711
 712         if (base->refcount_release != vmw_user_fence_base_release) {
 713                 pr_err("Invalid fence object handle 0x%08lx.\n",
 714                        (unsigned long)handle);
 715                 ttm_base_object_unref(&base);
 716                 return ERR_PTR(-EINVAL);
 717         }
 718
 719         return base;
 720 }
 721
 722
 723 int vmw_fence_obj_wait_ioctl(struct drm_device *dev, void *data,
 724                              struct drm_file *file_priv)
 725 {
 726         struct drm_vmw_fence_wait_arg *arg =
 727             (struct drm_vmw_fence_wait_arg *)data;
 728         unsigned long timeout;
 729         struct ttm_base_object *base;
 730         struct vmw_fence_obj *fence;
 731         struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
 732         int ret;
 733         uint64_t wait_timeout = ((uint64_t)arg->timeout_us * HZ);
 734
 735         /*
 736          * 64-bit division not present on 32-bit systems, so do an
 737          * approximation. (Divide by 1000000).
 738          */
 739
 740         wait_timeout = (wait_timeout >> 20) + (wait_timeout >> 24) -
 741           (wait_timeout >> 26);
 742
 743         if (!arg->cookie_valid) {
 744                 arg->cookie_valid = 1;
 745                 arg->kernel_cookie = jiffies + wait_timeout;
 746         }
 747
 748         base = vmw_fence_obj_lookup(tfile, arg->handle);
 749         if (IS_ERR(base))
 750                 return PTR_ERR(base);
 751
 752         fence = &(container_of(base, struct vmw_user_fence, base)->fence);
 753
 754         timeout = jiffies;
 755         if (time_after_eq(timeout, (unsigned long)arg->kernel_cookie)) {
 756                 ret = ((vmw_fence_obj_signaled(fence)) ?
 757                        0 : -EBUSY);
 758                 goto out;
 759         }
 760
 761         timeout = (unsigned long)arg->kernel_cookie - timeout;
 762
 763         ret = vmw_fence_obj_wait(fence, arg->lazy, true, timeout);
 764
 765 out:
 766         ttm_base_object_unref(&base);
 767
 768         /*
 769          * Optionally unref the fence object.
 770          */
 771
 772         if (ret == 0 && (arg->wait_options & DRM_VMW_WAIT_OPTION_UNREF))
 773                 return ttm_ref_object_base_unref(tfile, arg->handle);
 774         return ret;
 775 }
 776
 777 int vmw_fence_obj_signaled_ioctl(struct drm_device *dev, void *data,
 778                                  struct drm_file *file_priv)
 779 {
 780         struct drm_vmw_fence_signaled_arg *arg =
 781                 (struct drm_vmw_fence_signaled_arg *) data;
 782         struct ttm_base_object *base;
 783         struct vmw_fence_obj *fence;
 784         struct vmw_fence_manager *fman;
 785         struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
 786         struct vmw_private *dev_priv = vmw_priv(dev);
 787
 788         base = vmw_fence_obj_lookup(tfile, arg->handle);
 789         if (IS_ERR(base))
 790                 return PTR_ERR(base);
 791
 792         fence = &(container_of(base, struct vmw_user_fence, base)->fence);
 793         fman = fman_from_fence(fence);
 794
 795         arg->signaled = vmw_fence_obj_signaled(fence);
 796
 797         arg->signaled_flags = arg->flags;
 798         spin_lock(&fman->lock);
 799         arg->passed_seqno = dev_priv->last_read_seqno;
 800         spin_unlock(&fman->lock);
 801
 802         ttm_base_object_unref(&base);
 803
 804         return 0;
 805 }
 806
 807
 808 int vmw_fence_obj_unref_ioctl(struct drm_device *dev, void *data,
 809                               struct drm_file *file_priv)
 810 {
 811         struct drm_vmw_fence_arg *arg =
 812                 (struct drm_vmw_fence_arg *) data;
 813
 814         return ttm_ref_object_base_unref(vmw_fpriv(file_priv)->tfile,
 815                                          arg->handle);
 816 }
 817
 818 /**
 819  * vmw_event_fence_action_seq_passed
 820  *
 821  * @action: The struct vmw_fence_action embedded in a struct
 822  * vmw_event_fence_action.
 823  *
 824  * This function is called when the seqno of the fence where @action is
 825  * attached has passed. It queues the event on the submitter's event list.
 826  * This function is always called from atomic context.
 827  */
 828 static void vmw_event_fence_action_seq_passed(struct vmw_fence_action *action)
 829 {
 830         struct vmw_event_fence_action *eaction =
 831                 container_of(action, struct vmw_event_fence_action, action);
 832         struct drm_device *dev = eaction->dev;
 833         struct drm_pending_event *event = eaction->event;
 834
 835         if (unlikely(event == NULL))
 836                 return;
 837
 838         spin_lock_irq(&dev->event_lock);
 839
 840         if (likely(eaction->tv_sec != NULL)) {
 841                 struct timespec64 ts;
 842
 843                 ktime_get_ts64(&ts);
 844                 /* monotonic time, so no y2038 overflow */
 845                 *eaction->tv_sec = ts.tv_sec;
 846                 *eaction->tv_usec = ts.tv_nsec / NSEC_PER_USEC;
 847         }
 848
 849         drm_send_event_locked(dev, eaction->event);
 850         eaction->event = NULL;
 851         spin_unlock_irq(&dev->event_lock);
 852 }
 853
 854 /**
 855  * vmw_event_fence_action_cleanup
 856  *
 857  * @action: The struct vmw_fence_action embedded in a struct
 858  * vmw_event_fence_action.
 859  *
 860  * This function is the struct vmw_fence_action destructor. It's typically
 861  * called from a workqueue.
 862  */
 863 static void vmw_event_fence_action_cleanup(struct vmw_fence_action *action)
 864 {
 865         struct vmw_event_fence_action *eaction =
 866                 container_of(action, struct vmw_event_fence_action, action);
 867
 868         vmw_fence_obj_unreference(&eaction->fence);
 869         kfree(eaction);
 870 }
 871
 872
 873 /**
 874  * vmw_fence_obj_add_action - Add an action to a fence object.
 875  *
 876  * @fence: The fence object.
 877  * @action: The action to add.
 878  *
 879  * Note that the action callbacks may be executed before this function
 880  * returns.
 881  */
 882 static void vmw_fence_obj_add_action(struct vmw_fence_obj *fence,
 883                               struct vmw_fence_action *action)
 884 {
 885         struct vmw_fence_manager *fman = fman_from_fence(fence);
 886         bool run_update = false;
 887
 888         mutex_lock(&fman->goal_irq_mutex);
 889         spin_lock(&fman->lock);
 890
 891         fman->pending_actions[action->type]++;
 892         if (dma_fence_is_signaled_locked(&fence->base)) {
 893                 struct list_head action_list;
 894
 895                 INIT_LIST_HEAD(&action_list);
 896                 list_add_tail(&action->head, &action_list);
 897                 vmw_fences_perform_actions(fman, &action_list);
 898         } else {
 899                 list_add_tail(&action->head, &fence->seq_passed_actions);
 900
 901                 /*
 902                  * This function may set fman::seqno_valid, so it must
 903                  * be run with the goal_irq_mutex held.
 904                  */
 905                 run_update = vmw_fence_goal_check_locked(fence);
 906         }
 907
 908         spin_unlock(&fman->lock);
 909
 910         if (run_update) {
 911                 if (!fman->goal_irq_on) {
 912                         fman->goal_irq_on = true;
 913                         vmw_goal_waiter_add(fman->dev_priv);
 914                 }
 915                 vmw_fences_update(fman);
 916         }
 917         mutex_unlock(&fman->goal_irq_mutex);
 918
 919 }
 920
 921 /**
 922  * vmw_event_fence_action_queue - Post an event for sending when a fence
 923  * object seqno has passed.
 924  *
 925  * @file_priv: The file connection on which the event should be posted.
 926  * @fence: The fence object on which to post the event.
 927  * @event: Event to be posted. This event should've been alloced
 928  * using k[mz]alloc, and should've been completely initialized.
 929  * @tv_sec: If non-null, the variable pointed to will be assigned
 930  * current time tv_sec val when the fence signals.
 931  * @tv_usec: Must be set if @tv_sec is set, and the variable pointed to will
 932  * be assigned the current time tv_usec val when the fence signals.
 933  * @interruptible: Interruptible waits if possible.
 934  *
 935  * As a side effect, the object pointed to by @event may have been
 936  * freed when this function returns. If this function returns with
 937  * an error code, the caller needs to free that object.
 938  */
 939
 940 int vmw_event_fence_action_queue(struct drm_file *file_priv,
 941                                  struct vmw_fence_obj *fence,
 942                                  struct drm_pending_event *event,
 943                                  uint32_t *tv_sec,
 944                                  uint32_t *tv_usec,
 945                                  bool interruptible)
 946 {
 947         struct vmw_event_fence_action *eaction;
 948         struct vmw_fence_manager *fman = fman_from_fence(fence);
 949
 950         eaction = kzalloc(sizeof(*eaction), GFP_KERNEL);
 951         if (unlikely(!eaction))
 952                 return -ENOMEM;
 953
 954         eaction->event = event;
 955
 956         eaction->action.seq_passed = vmw_event_fence_action_seq_passed;
 957         eaction->action.cleanup = vmw_event_fence_action_cleanup;
 958         eaction->action.type = VMW_ACTION_EVENT;
 959
 960         eaction->fence = vmw_fence_obj_reference(fence);
 961         eaction->dev = &fman->dev_priv->drm;
 962         eaction->tv_sec = tv_sec;
 963         eaction->tv_usec = tv_usec;
 964
 965         vmw_fence_obj_add_action(fence, &eaction->action);
 966
 967         return 0;
 968 }
 969
 970 struct vmw_event_fence_pending {
 971         struct drm_pending_event base;
 972         struct drm_vmw_event_fence event;
 973 };
 974
 975 static int vmw_event_fence_action_create(struct drm_file *file_priv,
 976                                   struct vmw_fence_obj *fence,
 977                                   uint32_t flags,
 978                                   uint64_t user_data,
 979                                   bool interruptible)
 980 {
 981         struct vmw_event_fence_pending *event;
 982         struct vmw_fence_manager *fman = fman_from_fence(fence);
 983         struct drm_device *dev = &fman->dev_priv->drm;
 984         int ret;
 985
 986         event = kzalloc(sizeof(*event), GFP_KERNEL);
 987         if (unlikely(!event)) {
 988                 DRM_ERROR("Failed to allocate an event.\n");
 989                 ret = -ENOMEM;
 990                 goto out_no_space;
 991         }
 992
 993         event->event.base.type = DRM_VMW_EVENT_FENCE_SIGNALED;
 994         event->event.base.length = sizeof(*event);
 995         event->event.user_data = user_data;
 996
 997         ret = drm_event_reserve_init(dev, file_priv, &event->base, &event->event.base);
 998
 999         if (unlikely(ret != 0)) {
1000                 DRM_ERROR("Failed to allocate event space for this file.\n");
1001                 kfree(event);
1002                 goto out_no_space;
1003         }
1004
1005         if (flags & DRM_VMW_FE_FLAG_REQ_TIME)
1006                 ret = vmw_event_fence_action_queue(file_priv, fence,
1007                                                    &event->base,
1008                                                    &event->event.tv_sec,
1009                                                    &event->event.tv_usec,
1010                                                    interruptible);
1011         else
1012                 ret = vmw_event_fence_action_queue(file_priv, fence,
1013                                                    &event->base,
1014                                                    NULL,
1015                                                    NULL,
1016                                                    interruptible);
1017         if (ret != 0)
1018                 goto out_no_queue;
1019
1020         return 0;
1021
1022 out_no_queue:
1023         drm_event_cancel_free(dev, &event->base);
1024 out_no_space:
1025         return ret;
1026 }
1027
1028 int vmw_fence_event_ioctl(struct drm_device *dev, void *data,
1029                           struct drm_file *file_priv)
1030 {
1031         struct vmw_private *dev_priv = vmw_priv(dev);
1032         struct drm_vmw_fence_event_arg *arg =
1033                 (struct drm_vmw_fence_event_arg *) data;
1034         struct vmw_fence_obj *fence = NULL;
1035         struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv);
1036         struct ttm_object_file *tfile = vmw_fp->tfile;
1037         struct drm_vmw_fence_rep __user *user_fence_rep =
1038                 (struct drm_vmw_fence_rep __user *)(unsigned long)
1039                 arg->fence_rep;
1040         uint32_t handle;
1041         int ret;
1042
1043         /*
1044          * Look up an existing fence object,
1045          * and if user-space wants a new reference,
1046          * add one.
1047          */
1048         if (arg->handle) {
1049                 struct ttm_base_object *base =
1050                         vmw_fence_obj_lookup(tfile, arg->handle);
1051
1052                 if (IS_ERR(base))
1053                         return PTR_ERR(base);
1054
1055                 fence = &(container_of(base, struct vmw_user_fence,
1056                                        base)->fence);
1057                 (void) vmw_fence_obj_reference(fence);
1058
1059                 if (user_fence_rep != NULL) {
1060                         ret = ttm_ref_object_add(vmw_fp->tfile, base,
1061                                                  NULL, false);
1062                         if (unlikely(ret != 0)) {
1063                                 DRM_ERROR("Failed to reference a fence "
1064                                           "object.\n");
1065                                 goto out_no_ref_obj;
1066                         }
1067                         handle = base->handle;
1068                 }
1069                 ttm_base_object_unref(&base);
1070         }
1071
1072         /*
1073          * Create a new fence object.
1074          */
1075         if (!fence) {
1076                 ret = vmw_execbuf_fence_commands(file_priv, dev_priv,
1077                                                  &fence,
1078                                                  (user_fence_rep) ?
1079                                                  &handle : NULL);
1080                 if (unlikely(ret != 0)) {
1081                         DRM_ERROR("Fence event failed to create fence.\n");
1082                         return ret;
1083                 }
1084         }
1085
1086         BUG_ON(fence == NULL);
1087
1088         ret = vmw_event_fence_action_create(file_priv, fence,
1089                                             arg->flags,
1090                                             arg->user_data,
1091                                             true);
1092         if (unlikely(ret != 0)) {
1093                 if (ret != -ERESTARTSYS)
1094                         DRM_ERROR("Failed to attach event to fence.\n");
1095                 goto out_no_create;
1096         }
1097
1098         vmw_execbuf_copy_fence_user(dev_priv, vmw_fp, 0, user_fence_rep, fence,
1099                                     handle, -1);
1100         vmw_fence_obj_unreference(&fence);
1101         return 0;
1102 out_no_create:
1103         if (user_fence_rep != NULL)
1104                 ttm_ref_object_base_unref(tfile, handle);
1105 out_no_ref_obj:
1106         vmw_fence_obj_unreference(&fence);
1107         return ret;
1108 }