Re: [PATCH v5 1/6] drm/i915/skl: Add support for the SAGV, fix underrun hangs

On Tue, Aug 02, 2016 at 02:52:49PM -0400, Lyude wrote:

Since the watermark calculations for Skylake are still broken, we're apt to hitting underruns very easily under multi-monitor configurations. While it would be lovely if this was fixed, it's not. Another problem that's been coming from this however, is the mysterious issue of underruns causing full system hangs. An easy way to reproduce this with a skylake system:

• Get a laptop with a skylake GPU, and hook up two external monitors to it
• Move the cursor from the built-in LCD to one of the external displays as quickly as you can
• You'll get a few pipe underruns, and eventually the entire system will just freeze.

After doing a lot of investigation and reading through the bspec, I found the existence of the SAGV, which is responsible for adjusting the system agent voltage and clock frequencies depending on how much power we need. According to the bspec:

"The display engine access to system memory is blocked during the adjustment time. SAGV defaults to enabled. Software must use the GT-driver pcode mailbox to disable SAGV when the display engine is not able to tolerate the blocking time."

The rest of the bspec goes on to explain that software can simply leave the SAGV enabled, and disable it when we use interlaced pipes/have more then one pipe active.

Sure enough, with this patchset the system hangs resulting from pipe underruns on Skylake have completely vanished on my T460s. Additionally, the bspec mentions turning off the SAGV with more then one pipe enabled as a workaround for display underruns. While this patch doesn't entirely fix that, it looks like it does improve the situation a little bit so it's likely this is going to be required to make watermarks on Skylake fully functional.

Changes since v5:

• Don't use is_power_of_2. Makes things confusing
• Don't use the old state to figure out whether or not to enable/disable the sagv, use the new one
• Split the loop in skl_disable_sagv into it's own function

Changes since v4:

• Use is_power_of_2 against active_crtcs to check whether we have > 1 pipe enabled
• Fix skl_sagv_get_hw_state(): (temp & 0x1) indicates disabled, 0x0 enabled
• Call skl_sagv_enable/disable() from pre/post-plane updates

Changes since v3:

• Use time_before() to compare timeout to jiffies

Changes since v2:

• Really apply minor style nitpicks to patch this time

Changes since v1:

• Added comments about this probably being one of the requirements to fixing Skylake's watermark issues
• Minor style nitpicks from Matt Roper
• Disable these functions on Broxton, since it doesn't have an SAGV

Reviewed-by: Matt Roper matthew.d.roper@intel.com Signed-off-by: Lyude cpaul@redhat.com Cc: Daniel Vetter daniel.vetter@ffwll.ch Cc: Ville Syrjälä ville.syrjala@linux.intel.com Cc: stable@vger.kernel.org

---

drivers/gpu/drm/i915/i915_drv.h | 2 + drivers/gpu/drm/i915/i915_reg.h | 5 ++ drivers/gpu/drm/i915/intel_display.c | 12 ++++ drivers/gpu/drm/i915/intel_drv.h | 2 + drivers/gpu/drm/i915/intel_pm.c | 112 +++++++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+)

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 65ada5d..87018d3 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -1962,6 +1962,8 @@ struct drm_i915_private { struct i915_suspend_saved_registers regfile; struct vlv_s0ix_state vlv_s0ix_state;

• bool skl_sagv_enabled;
• struct { /*
  • Raw watermark latency values:

diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 2f93d4a..5fb1c63 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -7170,6 +7170,11 @@ enum { #define HSW_PCODE_DE_WRITE_FREQ_REQ 0x17 #define DISPLAY_IPS_CONTROL 0x19 #define HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL 0x1A +#define GEN9_PCODE_SAGV_CONTROL 0x21 +#define GEN9_SAGV_DISABLE 0x0 +#define GEN9_SAGV_LOW_FREQ 0x1 +#define GEN9_SAGV_HIGH_FREQ 0x2 +#define GEN9_SAGV_DYNAMIC_FREQ 0x3 #define GEN6_PCODE_DATA _MMIO(0x138128) #define GEN6_PCODE_FREQ_IA_RATIO_SHIFT 8 #define GEN6_PCODE_FREQ_RING_RATIO_SHIFT 16 diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index a8e8cc8..76ba79f 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -4565,6 +4565,7 @@ static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state) struct intel_crtc_state *pipe_config = to_intel_crtc_state(crtc->base.state); struct drm_device *dev = crtc->base.dev;

• struct drm_i915_private *dev_priv = to_i915(dev); struct drm_plane *primary = crtc->base.primary; struct drm_plane_state *old_pri_state = drm_atomic_get_existing_plane_state(old_state, primary);

@@ -4589,6 +4590,9 @@ static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state) !old_primary_state->visible)) intel_post_enable_primary(&crtc->base); }

• if (hweight32(dev_priv->active_crtcs) <= 1)

• skl_enable_sagv(dev_priv);
  

There might be a slightly better place to handle this; see comment below.

}

static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state) @@ -4649,6 +4653,14 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state) }

/*

• * SKL workaround: bspec recommends we disable the SAGV when we have
  

• * more then one pipe enabled
  

• */
  

• if (pipe_config->base.active &&

•    hweight32(dev_priv->active_crtcs | drm_crtc_mask(&crtc->base)) > 1)
  

• skl_disable_sagv(dev_priv);
  

As Hans pointed out, this doesn't look right. I'm guessing you're trying to guard against the case where there's no intersection between the starting pipe usage and the final pipe usage? E.g., only pipe A active before commit, only pipe B active after commit, and you're worried there might be a brief point where both A and B are on together? I don't think this should really matter since we power everything old-changing down before powering anything new-changing up. So I think all we really need to care about for enabling/disabling is how many CRTC's there are in the final state.

It seems like a more natural place to place to handle enable/disable would be in the 'if (intel_state->modeset)' blocks of intel_atomic_commit_tail() (since only a modeset operation could change the number of CRTC's in use to trigger a SAGV toggle). That also has the slight benefit of only getting run once for the atomic transaction rather than once for each CRTC (and you can just do a simple test of hweight(intel_state->active_crtcs) to figure out how many CRTC's are going to be on after the transaction completes.

• /*
  • If we're doing a modeset, we're done. No need to do any pre-vblank
  • watermark programming here.
  */

diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h index 50cdc89..6b0532a 100644 --- a/drivers/gpu/drm/i915/intel_drv.h +++ b/drivers/gpu/drm/i915/intel_drv.h @@ -1709,6 +1709,8 @@ void ilk_wm_get_hw_state(struct drm_device *dev); void skl_wm_get_hw_state(struct drm_device *dev); void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv, struct skl_ddb_allocation *ddb /* out */); +int skl_enable_sagv(struct drm_i915_private *dev_priv); +int skl_disable_sagv(struct drm_i915_private *dev_priv); uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config); bool ilk_disable_lp_wm(struct drm_device *dev); int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6); diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index f610b71..68721a5 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -2884,6 +2884,116 @@ skl_wm_plane_id(const struct intel_plane *plane) }

static void +skl_sagv_get_hw_state(struct drm_i915_private *dev_priv) +{

• u32 temp;
• int ret;
• if (IS_BROXTON(dev_priv))

• return;
  

• mutex_lock(&dev_priv->rps.hw_lock);
• ret = sandybridge_pcode_read(dev_priv, GEN9_PCODE_SAGV_CONTROL, &temp);
• mutex_unlock(&dev_priv->rps.hw_lock);
• if (!ret) {

• dev_priv->skl_sagv_enabled = !(temp & 0x1);
  

• } else {

• /*
  

•  * If for some reason we can't access the SAGV state, follow
  

•  * the bspec and assume it's enabled
  

•  */
  

• DRM_ERROR("Failed to get SAGV state, assuming enabled\n");
  

• dev_priv->skl_sagv_enabled = true;
  

• }

+}

+/*

• • SAGV dynamically adjusts the system agent voltage and clock frequencies
• • depending on power and performance requirements. The display engine access
• • to system memory is blocked during the adjustment time. Having this enabled
• • in multi-pipe configurations can cause issues (such as underruns causing
• • full system hangs), and the bspec also suggests that software disable it
• • when more then one pipe is enabled.
• */

+int +skl_enable_sagv(struct drm_i915_private *dev_priv) +{

• int ret;
• if (IS_BROXTON(dev_priv))

• return 0;
  

• if (dev_priv->skl_sagv_enabled)

• return 0;
  

• mutex_lock(&dev_priv->rps.hw_lock);
• DRM_DEBUG_KMS("Enabling the SAGV\n");
• ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,

• 		      GEN9_SAGV_DYNAMIC_FREQ);
  

• if (!ret)

• dev_priv->skl_sagv_enabled = true;
  

• else

• DRM_ERROR("Failed to enable the SAGV\n");
  

• /* We don't need to wait for SAGV when enabling */
• mutex_unlock(&dev_priv->rps.hw_lock);
• return ret;

+}

+static int +skl_do_sagv_disable(struct drm_i915_private *dev_priv) +{

• int ret;
• uint32_t temp;
• ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,

• 		      GEN9_SAGV_DISABLE);
  

• if (ret) {

• DRM_ERROR("Failed to disable the SAGV\n");
  

• return ret;
  

• }
• ret = sandybridge_pcode_read(dev_priv, GEN9_PCODE_SAGV_CONTROL,

• 		     &temp);
  

• if (ret) {

• DRM_ERROR("Failed to check the status of the SAGV\n");
  

• return ret;
  

• }
• return temp & 0x1;

+}

+int +skl_disable_sagv(struct drm_i915_private *dev_priv) +{

• int ret, result;
• if (IS_BROXTON(dev_priv))

• return 0;
  

• if (!dev_priv->skl_sagv_enabled)

• return 0;
  

• mutex_lock(&dev_priv->rps.hw_lock);
• DRM_DEBUG_KMS("Disabling the SAGV\n");
• /* bspec says to keep retrying for at least 1 ms */
• ret = wait_for(result = skl_do_sagv_disable(dev_priv), 1);
• mutex_unlock(&dev_priv->rps.hw_lock);
• if (ret == -ETIMEDOUT)

• DRM_ERROR("Request to disable SAGV timed out\n");
  

• else {

Minor style nitpick; if either branch of an if/else needs braces, they both need braces.

• if (result == 1)
  

You've got a few uses of 0x1 as a magic number that are correct, but seem counterintuitive to someone not looking at the bspec (1=off). Maybe replacing these with a #define might help clarify slightly?

Matt

• 	dev_priv->skl_sagv_enabled = false;
  

• ret = result;
  

• }
• return ret;

+}

+static void skl_ddb_get_pipe_allocation_limits(struct drm_device *dev, const struct intel_crtc_state *cstate, struct skl_ddb_entry *alloc, /* out */ @@ -4236,6 +4346,8 @@ void skl_wm_get_hw_state(struct drm_device *dev) /* Easy/common case; just sanitize DDB now if everything off */ memset(ddb, 0, sizeof(*ddb)); }

• skl_sagv_get_hw_state(dev_priv);

}

static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)

2.7.4