This modifies the datatypes used by the vblank code to provide both 64 bits of vblank count and switch to using ktime_t for timestamps to increase resolution from microseconds to nanoseconds.
The driver interfaces have been left using 32 bits of vblank count; all of the code necessary to widen that value for the user API was already included to handle devices returning fewer than 32-bits.
This will provide the necessary datatypes for the Vulkan API.
v2:
* Re-write wait_vblank ioctl to ABSOLUTE sequence
When an application uses the WAIT_VBLANK ioctl with RELATIVE or NEXTONMISS bits set, the target vblank interval is updated within the kernel. We need to write that target back to the ioctl buffer and update the flags bits so that if the wait is interrupted by a signal, when it is re-started, it will target precisely the same vblank count as before.
* Leave driver API with 32-bit vblank count
Suggested-by: Michel Dänzer michel@daenzer.net Suggested-by: Daniel Vetter daniel@ffwll.ch Signed-off-by: Keith Packard keithp@keithp.com --- drivers/gpu/drm/drm_vblank.c | 186 +++++++++++++++++++++++++------------------ include/drm/drmP.h | 2 +- include/drm/drm_drv.h | 2 +- include/drm/drm_vblank.h | 16 ++-- 4 files changed, 120 insertions(+), 86 deletions(-)
diff --git a/drivers/gpu/drm/drm_vblank.c b/drivers/gpu/drm/drm_vblank.c index 463e4d81fb0d..346601ad698d 100644 --- a/drivers/gpu/drm/drm_vblank.c +++ b/drivers/gpu/drm/drm_vblank.c @@ -43,7 +43,7 @@
static bool drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe, - struct timeval *tvblank, bool in_vblank_irq); + ktime_t *tvblank, bool in_vblank_irq);
static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */
@@ -64,7 +64,7 @@ MODULE_PARM_DESC(timestamp_monotonic, "Use monotonic timestamps");
static void store_vblank(struct drm_device *dev, unsigned int pipe, u32 vblank_count_inc, - struct timeval *t_vblank, u32 last) + ktime_t t_vblank, u32 last) { struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
@@ -73,7 +73,7 @@ static void store_vblank(struct drm_device *dev, unsigned int pipe, vblank->last = last;
write_seqlock(&vblank->seqlock); - vblank->time = *t_vblank; + vblank->time = t_vblank; vblank->count += vblank_count_inc; write_sequnlock(&vblank->seqlock); } @@ -116,7 +116,7 @@ static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe { u32 cur_vblank; bool rc; - struct timeval t_vblank; + ktime_t t_vblank; int count = DRM_TIMESTAMP_MAXRETRIES;
spin_lock(&dev->vblank_time_lock); @@ -136,13 +136,13 @@ static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid. */ if (!rc) - t_vblank = (struct timeval) {0, 0}; + t_vblank = 0;
/* * +1 to make sure user will never see the same * vblank counter value before and after a modeset */ - store_vblank(dev, pipe, 1, &t_vblank, cur_vblank); + store_vblank(dev, pipe, 1, t_vblank, cur_vblank);
spin_unlock(&dev->vblank_time_lock); } @@ -165,7 +165,7 @@ static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe, struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; u32 cur_vblank, diff; bool rc; - struct timeval t_vblank; + ktime_t t_vblank; int count = DRM_TIMESTAMP_MAXRETRIES; int framedur_ns = vblank->framedur_ns;
@@ -190,11 +190,9 @@ static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe, /* trust the hw counter when it's around */ diff = (cur_vblank - vblank->last) & dev->max_vblank_count; } else if (rc && framedur_ns) { - const struct timeval *t_old; - u64 diff_ns; + ktime_t diff_ns;
- t_old = &vblank->time; - diff_ns = timeval_to_ns(&t_vblank) - timeval_to_ns(t_old); + diff_ns = t_vblank - vblank->time;
/* * Figure out how many vblanks we've missed based @@ -228,7 +226,7 @@ static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe, }
DRM_DEBUG_VBL("updating vblank count on crtc %u:" - " current=%u, diff=%u, hw=%u hw_last=%u\n", + " current=%llu, diff=%u, hw=%u hw_last=%u\n", pipe, vblank->count, diff, cur_vblank, vblank->last);
if (diff == 0) { @@ -243,9 +241,9 @@ static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe, * for now, to mark the vblanktimestamp as invalid. */ if (!rc && in_vblank_irq) - t_vblank = (struct timeval) {0, 0}; + t_vblank = 0;
- store_vblank(dev, pipe, diff, &t_vblank, cur_vblank); + store_vblank(dev, pipe, diff, t_vblank, cur_vblank); }
static u32 drm_vblank_count(struct drm_device *dev, unsigned int pipe) @@ -567,10 +565,10 @@ EXPORT_SYMBOL(drm_calc_timestamping_constants); bool drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, unsigned int pipe, int *max_error, - struct timeval *vblank_time, + ktime_t *vblank_time, bool in_vblank_irq) { - struct timeval tv_etime; + ktime_t prev_etime; ktime_t stime, etime; bool vbl_status; struct drm_crtc *crtc; @@ -663,29 +661,26 @@ bool drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, etime = ktime_mono_to_real(etime);
/* save this only for debugging purposes */ - tv_etime = ktime_to_timeval(etime); + prev_etime = etime; /* Subtract time delta from raw timestamp to get final * vblank_time timestamp for end of vblank. */ etime = ktime_sub_ns(etime, delta_ns); - *vblank_time = ktime_to_timeval(etime); + *vblank_time = etime;
- DRM_DEBUG_VBL("crtc %u : v p(%d,%d)@ %ld.%ld -> %ld.%ld [e %d us, %d rep]\n", + DRM_DEBUG_VBL("crtc %u : v p(%d,%d)@ %lld -> %lld [e %d us, %d rep]\n", pipe, hpos, vpos, - (long)tv_etime.tv_sec, (long)tv_etime.tv_usec, - (long)vblank_time->tv_sec, (long)vblank_time->tv_usec, + (long long) prev_etime, + (long long) etime, duration_ns/1000, i);
return true; } EXPORT_SYMBOL(drm_calc_vbltimestamp_from_scanoutpos);
-static struct timeval get_drm_timestamp(void) +static ktime_t get_drm_timestamp(void) { - ktime_t now; - - now = drm_timestamp_monotonic ? ktime_get() : ktime_get_real(); - return ktime_to_timeval(now); + return drm_timestamp_monotonic ? ktime_get() : ktime_get_real(); }
/** @@ -711,7 +706,7 @@ static struct timeval get_drm_timestamp(void) */ static bool drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe, - struct timeval *tvblank, bool in_vblank_irq) + ktime_t *tvblank, bool in_vblank_irq) { bool ret = false;
@@ -743,7 +738,7 @@ drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe, * Returns: * The software vblank counter. */ -u32 drm_crtc_vblank_count(struct drm_crtc *crtc) +u64 drm_crtc_vblank_count(struct drm_crtc *crtc) { return drm_vblank_count(crtc->dev, drm_crtc_index(crtc)); } @@ -763,15 +758,15 @@ EXPORT_SYMBOL(drm_crtc_vblank_count); * * This is the legacy version of drm_crtc_vblank_count_and_time(). */ -static u32 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe, - struct timeval *vblanktime) +static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe, + ktime_t *vblanktime) { struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; - u32 vblank_count; + u64 vblank_count; unsigned int seq;
if (WARN_ON(pipe >= dev->num_crtcs)) { - *vblanktime = (struct timeval) { 0 }; + *vblanktime = 0; return 0; }
@@ -795,8 +790,8 @@ static u32 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe, * modesetting activity. Returns corresponding system timestamp of the time * of the vblank interval that corresponds to the current vblank counter value. */ -u32 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc, - struct timeval *vblanktime) +u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc, + ktime_t *vblanktime) { return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc), vblanktime); @@ -805,11 +800,14 @@ EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
static void send_vblank_event(struct drm_device *dev, struct drm_pending_vblank_event *e, - unsigned long seq, struct timeval *now) + u64 seq, ktime_t now) { + struct timeval tv; + + tv = ktime_to_timeval(now); e->event.sequence = seq; - e->event.tv_sec = now->tv_sec; - e->event.tv_usec = now->tv_usec; + e->event.tv_sec = tv.tv_sec; + e->event.tv_usec = tv.tv_usec;
trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, e->event.sequence); @@ -864,7 +862,7 @@ void drm_crtc_arm_vblank_event(struct drm_crtc *crtc, assert_spin_locked(&dev->event_lock);
e->pipe = pipe; - e->event.sequence = drm_vblank_count(dev, pipe); + e->sequence = drm_vblank_count(dev, pipe); e->event.crtc_id = crtc->base.id; list_add_tail(&e->base.link, &dev->vblank_event_list); } @@ -885,19 +883,19 @@ void drm_crtc_send_vblank_event(struct drm_crtc *crtc, struct drm_pending_vblank_event *e) { struct drm_device *dev = crtc->dev; - unsigned int seq, pipe = drm_crtc_index(crtc); - struct timeval now; + u64 seq; + unsigned int pipe = drm_crtc_index(crtc); + ktime_t now;
if (dev->num_crtcs > 0) { seq = drm_vblank_count_and_time(dev, pipe, &now); } else { seq = 0; - now = get_drm_timestamp(); } e->pipe = pipe; e->event.crtc_id = crtc->base.id; - send_vblank_event(dev, e, seq, &now); + send_vblank_event(dev, e, seq, now); } EXPORT_SYMBOL(drm_crtc_send_vblank_event);
@@ -1124,9 +1122,9 @@ void drm_crtc_vblank_off(struct drm_crtc *crtc) unsigned int pipe = drm_crtc_index(crtc); struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; struct drm_pending_vblank_event *e, *t; - struct timeval now; + ktime_t now; unsigned long irqflags; - unsigned int seq; + u64 seq;
if (WARN_ON(pipe >= dev->num_crtcs)) return; @@ -1161,11 +1159,11 @@ void drm_crtc_vblank_off(struct drm_crtc *crtc) if (e->pipe != pipe) continue; DRM_DEBUG("Sending premature vblank event on disable: " - "wanted %u, current %u\n", - e->event.sequence, seq); + "wanted %llu current %llu\n", + e->sequence, seq); list_del(&e->base.link); drm_vblank_put(dev, pipe); - send_vblank_event(dev, e, seq, &now); + send_vblank_event(dev, e, seq, now); } spin_unlock_irqrestore(&dev->event_lock, irqflags);
@@ -1331,20 +1329,21 @@ int drm_legacy_modeset_ctl(struct drm_device *dev, void *data, return 0; }
-static inline bool vblank_passed(u32 seq, u32 ref) +static inline bool vblank_passed(u64 seq, u64 ref) { return (seq - ref) <= (1 << 23); }
static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe, + u64 req_seq, union drm_wait_vblank *vblwait, struct drm_file *file_priv) { struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; struct drm_pending_vblank_event *e; - struct timeval now; + ktime_t now; unsigned long flags; - unsigned int seq; + u64 seq; int ret;
e = kzalloc(sizeof(*e), GFP_KERNEL); @@ -1379,21 +1378,20 @@ static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
seq = drm_vblank_count_and_time(dev, pipe, &now);
- DRM_DEBUG("event on vblank count %u, current %u, crtc %u\n", - vblwait->request.sequence, seq, pipe); + DRM_DEBUG("event on vblank count %llu, current %llu, crtc %u\n", + req_seq, seq, pipe);
- trace_drm_vblank_event_queued(file_priv, pipe, - vblwait->request.sequence); + trace_drm_vblank_event_queued(file_priv, pipe, req_seq);
- e->event.sequence = vblwait->request.sequence; - if (vblank_passed(seq, vblwait->request.sequence)) { + e->sequence = req_seq; + if (vblank_passed(seq, req_seq)) { drm_vblank_put(dev, pipe); - send_vblank_event(dev, e, seq, &now); + send_vblank_event(dev, e, seq, now); vblwait->reply.sequence = seq; } else { /* drm_handle_vblank_events will call drm_vblank_put */ list_add_tail(&e->base.link, &dev->vblank_event_list); - vblwait->reply.sequence = vblwait->request.sequence; + vblwait->reply.sequence = req_seq; }
spin_unlock_irqrestore(&dev->event_lock, flags); @@ -1420,6 +1418,27 @@ static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait) }
/* + * Widen a 32-bit param to 64-bits. + * + * \param narrow 32-bit value (missing upper 32 bits) + * \param near 64-bit value that should be 'close' to near + * + * This function returns a 64-bit value using the lower 32-bits from + * 'narrow' and constructing the upper 32-bits so that the result is + * as close as possible to 'near'. + */ + +static u64 widen_32_to_64(u32 narrow, u64 near) +{ + u64 wide = narrow | (near & 0xffffffff00000000ULL); + if ((int64_t) (wide - near) > 0x80000000LL) + wide -= 0x100000000ULL; + else if ((int64_t) (near - wide) > 0x80000000LL) + wide += 0x100000000ULL; + return wide; +} + +/* * Wait for VBLANK. * * \param inode device inode. @@ -1439,6 +1458,7 @@ int drm_wait_vblank(struct drm_device *dev, void *data, struct drm_vblank_crtc *vblank; union drm_wait_vblank *vblwait = data; int ret; + u64 req_seq; unsigned int flags, seq, pipe, high_pipe;
if (!dev->irq_enabled) @@ -1474,12 +1494,14 @@ int drm_wait_vblank(struct drm_device *dev, void *data, if (dev->vblank_disable_immediate && drm_wait_vblank_is_query(vblwait) && READ_ONCE(vblank->enabled)) { - struct timeval now; + ktime_t now; + struct timeval tv;
vblwait->reply.sequence = drm_vblank_count_and_time(dev, pipe, &now); - vblwait->reply.tval_sec = now.tv_sec; - vblwait->reply.tval_usec = now.tv_usec; + tv = ktime_to_timeval(now); + vblwait->reply.tval_sec = tv.tv_sec; + vblwait->reply.tval_usec = tv.tv_usec; return 0; }
@@ -1492,9 +1514,12 @@ int drm_wait_vblank(struct drm_device *dev, void *data,
switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) { case _DRM_VBLANK_RELATIVE: - vblwait->request.sequence += seq; + req_seq = seq + vblwait->request.sequence; vblwait->request.type &= ~_DRM_VBLANK_RELATIVE; + vblwait->request.sequence = req_seq; + break; case _DRM_VBLANK_ABSOLUTE: + req_seq = widen_32_to_64(vblwait->request.sequence, seq); break; default: ret = -EINVAL; @@ -1502,31 +1527,36 @@ int drm_wait_vblank(struct drm_device *dev, void *data, }
if ((flags & _DRM_VBLANK_NEXTONMISS) && - vblank_passed(seq, vblwait->request.sequence)) - vblwait->request.sequence = seq + 1; + vblank_passed(seq, req_seq)) { + req_seq = seq + 1; + vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS; + vblwait->request.sequence = req_seq; + }
if (flags & _DRM_VBLANK_EVENT) { /* must hold on to the vblank ref until the event fires * drm_vblank_put will be called asynchronously */ - return drm_queue_vblank_event(dev, pipe, vblwait, file_priv); + return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv); }
- if (vblwait->request.sequence != seq) { - DRM_DEBUG("waiting on vblank count %u, crtc %u\n", - vblwait->request.sequence, pipe); + if (req_seq != seq) { + DRM_DEBUG("waiting on vblank count %llu, crtc %u\n", + req_seq, pipe); DRM_WAIT_ON(ret, vblank->queue, 3 * HZ, vblank_passed(drm_vblank_count(dev, pipe), - vblwait->request.sequence) || + req_seq) || !READ_ONCE(vblank->enabled)); }
if (ret != -EINTR) { - struct timeval now; + ktime_t now; + struct timeval tv;
vblwait->reply.sequence = drm_vblank_count_and_time(dev, pipe, &now); - vblwait->reply.tval_sec = now.tv_sec; - vblwait->reply.tval_usec = now.tv_usec; + tv = ktime_to_timeval(now); + vblwait->reply.tval_sec = tv.tv_sec; + vblwait->reply.tval_usec = tv.tv_usec;
DRM_DEBUG("crtc %d returning %u to client\n", pipe, vblwait->reply.sequence); @@ -1542,8 +1572,8 @@ int drm_wait_vblank(struct drm_device *dev, void *data, static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe) { struct drm_pending_vblank_event *e, *t; - struct timeval now; - unsigned int seq; + ktime_t now; + u64 seq;
assert_spin_locked(&dev->event_lock);
@@ -1552,15 +1582,15 @@ static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe) list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) { if (e->pipe != pipe) continue; - if (!vblank_passed(seq, e->event.sequence)) + if (!vblank_passed(seq, e->sequence)) continue;
- DRM_DEBUG("vblank event on %u, current %u\n", - e->event.sequence, seq); + DRM_DEBUG("vblank event on %llu, current %llu\n", + e->sequence, seq);
list_del(&e->base.link); drm_vblank_put(dev, pipe); - send_vblank_event(dev, e, seq, &now); + send_vblank_event(dev, e, seq, now); }
trace_drm_vblank_event(pipe, seq); diff --git a/include/drm/drmP.h b/include/drm/drmP.h index 39df16af7a4a..e50cf152f565 100644 --- a/include/drm/drmP.h +++ b/include/drm/drmP.h @@ -403,7 +403,7 @@ struct drm_device { spinlock_t vblank_time_lock; /**< Protects vblank count and time updates during vblank enable/disable */ spinlock_t vbl_lock;
- u32 max_vblank_count; /**< size of vblank counter register */ + u64 max_vblank_count; /**< size of vblank counter register */
/** * List of events diff --git a/include/drm/drm_drv.h b/include/drm/drm_drv.h index d855f9ae41a8..2e4e425b5fba 100644 --- a/include/drm/drm_drv.h +++ b/include/drm/drm_drv.h @@ -325,7 +325,7 @@ struct drm_driver { */ bool (*get_vblank_timestamp) (struct drm_device *dev, unsigned int pipe, int *max_error, - struct timeval *vblank_time, + ktime_t *vblank_time, bool in_vblank_irq);
/** diff --git a/include/drm/drm_vblank.h b/include/drm/drm_vblank.h index 4cde47332dfa..e809ab244919 100644 --- a/include/drm/drm_vblank.h +++ b/include/drm/drm_vblank.h @@ -48,6 +48,10 @@ struct drm_pending_vblank_event { */ unsigned int pipe; /** + * @sequence: frame event should be triggered at + */ + u64 sequence; + /** * @event: Actual event which will be sent to userspace. */ struct drm_event_vblank event; @@ -88,11 +92,11 @@ struct drm_vblank_crtc { /** * @count: Current software vblank counter. */ - u32 count; + u64 count; /** * @time: Vblank timestamp corresponding to @count. */ - struct timeval time; + ktime_t time;
/** * @refcount: Number of users/waiters of the vblank interrupt. Only when @@ -152,9 +156,9 @@ struct drm_vblank_crtc { };
int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs); -u32 drm_crtc_vblank_count(struct drm_crtc *crtc); -u32 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc, - struct timeval *vblanktime); +u64 drm_crtc_vblank_count(struct drm_crtc *crtc); +u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc, + ktime_t *vblanktime); void drm_crtc_send_vblank_event(struct drm_crtc *crtc, struct drm_pending_vblank_event *e); void drm_crtc_arm_vblank_event(struct drm_crtc *crtc, @@ -173,7 +177,7 @@ u32 drm_accurate_vblank_count(struct drm_crtc *crtc);
bool drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, unsigned int pipe, int *max_error, - struct timeval *vblank_time, + ktime_t *vblank_time, bool in_vblank_irq); void drm_calc_timestamping_constants(struct drm_crtc *crtc, const struct drm_display_mode *mode);