Commit e133e737 didn't correctly fix the integer overflow issue.

- unsigned int required_size; + u64 required_size; ... required_size = mode_cmd->pitch * mode_cmd->height; - if (unlikely(required_size > dev_priv->vram_size)) { + if (unlikely(required_size > (u64) dev_priv->vram_size)) {

Note that both pitch and height are u32. Their product is still u32 and would overflow before being assigned to required_size. A correct way is to convert pitch and height to u64 before the multiplication.

required_size = (u64)mode_cmd->pitch * (u64)mode_cmd->height;

This patch calls the existing vmw_kms_validate_mode_vram() for validation.

Signed-off-by: Xi Wang xi.wang@gmail.com --- drivers/gpu/drm/vmwgfx/vmwgfx_kms.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c index 8aa1dbb..f94b33a 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c @@ -1093,7 +1093,6 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev, struct vmw_surface *surface = NULL; struct vmw_dma_buffer *bo = NULL; struct ttm_base_object *user_obj; - u64 required_size; int ret;

/** @@ -1102,8 +1101,9 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev, * requested framebuffer. */

- required_size = mode_cmd->pitch * mode_cmd->height; - if (unlikely(required_size > (u64) dev_priv->vram_size)) { + if (!vmw_kms_validate_mode_vram(dev_priv, + mode_cmd->pitch, + mode_cmd->height)) { DRM_ERROR("VRAM size is too small for requested mode.\n"); return ERR_PTR(-ENOMEM); }

Reviewed- and tested by

Thomas Hellstrom thellstrom@vmware.com

Thanks, Thomas

On 12/21/2011 11:18 AM, Xi Wang wrote:

Commit e133e737 didn't correctly fix the integer overflow issue.

• unsigned int required_size;

• u64 required_size; ... required_size = mode_cmd->pitch * mode_cmd->height;

• if (unlikely(required_size> dev_priv->vram_size)) {

• if (unlikely(required_size> (u64) dev_priv->vram_size)) {

Note that both pitch and height are u32. Their product is still u32 and would overflow before being assigned to required_size. A correct way is to convert pitch and height to u64 before the multiplication.

required_size = (u64)mode_cmd->pitch * (u64)mode_cmd->height;

This patch calls the existing vmw_kms_validate_mode_vram() for validation.

Signed-off-by: Xi Wangxi.wang@gmail.com

drivers/gpu/drm/vmwgfx/vmwgfx_kms.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c index 8aa1dbb..f94b33a 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c @@ -1093,7 +1093,6 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev, struct vmw_surface *surface = NULL; struct vmw_dma_buffer *bo = NULL; struct ttm_base_object *user_obj;

• u64 required_size; int ret;

  /**

@@ -1102,8 +1101,9 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev, * requested framebuffer. */

• required_size = mode_cmd->pitch * mode_cmd->height;
• if (unlikely(required_size> (u64) dev_priv->vram_size)) {

• if (!vmw_kms_validate_mode_vram(dev_priv,

• 			mode_cmd->pitch,
  

• 			mode_cmd->height)) {
  

  DRM_ERROR("VRAM size is too small for requested mode.\n"); return ERR_PTR(-ENOMEM); }