diff --git a/src/3d/qgscameracontroller.cpp b/src/3d/qgscameracontroller.cpp
index 7e4a201f1bc7..4ea1fda6fd62 100644
--- a/src/3d/qgscameracontroller.cpp
+++ b/src/3d/qgscameracontroller.cpp
@@ -237,7 +237,7 @@ void QgsCameraController::readXml( const QDomElement &elem )
   setLookingAtPoint( QgsVector3D( x, elev, y ), dist, pitch, yaw );
 }
 
-double QgsCameraController::sampleDepthBuffer( const QImage &buffer, int px, int py )
+double QgsCameraController::sampleDepthBuffer( int px, int py )
 {
   double depth = 1;
 
@@ -246,9 +246,9 @@ double QgsCameraController::sampleDepthBuffer( const QImage &buffer, int px, int
   {
     for ( int y = py - 3; y <= py + 3; ++y )
     {
-      if ( buffer.valid( x, y ) )
+      if ( mDepthBufferImage.valid( x, y ) )
       {
-        depth = std::min( depth, Qgs3DUtils::decodeDepth( buffer.pixel( x, y ) ) );
+        depth = std::min( depth, Qgs3DUtils::decodeDepth( mDepthBufferImage.pixel( x, y ) ) );
       }
     }
   }
@@ -256,14 +256,18 @@ double QgsCameraController::sampleDepthBuffer( const QImage &buffer, int px, int
   if ( depth < 1 )
     return depth;
 
+  // Cache the computed depth, since averaging over all pixels can be expensive
+  if ( mDepthBufferNonVoidAverage )
+    return mDepthBufferNonVoidAverage.data();
+
   // Returns the average of depth values that are not 1 (void area)
   depth = 0;
   int samplesCount = 0;
-  for ( int x = 0; x < buffer.width(); ++x )
+  for ( int x = 0; x < mDepthBufferImage.width(); ++x )
   {
-    for ( int y = 0; y < buffer.height(); ++y )
+    for ( int y = 0; y < mDepthBufferImage.height(); ++y )
     {
-      double d = Qgs3DUtils::decodeDepth( buffer.pixel( x, y ) );
+      double d = Qgs3DUtils::decodeDepth( mDepthBufferImage.pixel( x, y ) );
       if ( d < 1 )
       {
         depth += d;
@@ -278,6 +282,8 @@ double QgsCameraController::sampleDepthBuffer( const QImage &buffer, int px, int
   else
     depth /= samplesCount;
 
+  mDepthBufferNonVoidAverage = depth;
+
   return depth;
 }
 
@@ -316,7 +322,7 @@ void QgsCameraController::onPositionChanged( Qt3DInput::QMouseEvent *mouse )
 
 bool QgsCameraController::screenPointToWorldPos( QPoint position, Qt3DRender::QCamera *mCameraBefore, double &depth, QVector3D &worldPosition )
 {
-  depth = sampleDepthBuffer( mDepthBufferImage, position.x(), position.y() );
+  depth = sampleDepthBuffer( position.x(), position.y() );
   if ( !std::isfinite( depth ) )
   {
     QgsDebugMsgLevel( QStringLiteral( "screenPointToWorldPos: depth is NaN or Inf. This should not happen." ), 2 );
@@ -1023,6 +1029,7 @@ void QgsCameraController::depthBufferCaptured( const QImage &depthImage )
 {
   mDepthBufferImage = depthImage;
   mDepthBufferIsReady = true;
+  mDepthBufferNonVoidAverage.setEnabled( false );
 
   if ( mCurrentOperation == MouseOperation::ZoomWheel )
   {
diff --git a/src/3d/qgscameracontroller.h b/src/3d/qgscameracontroller.h
index 08830805c12e..c8558a6a20a4 100644
--- a/src/3d/qgscameracontroller.h
+++ b/src/3d/qgscameracontroller.h
@@ -17,6 +17,7 @@
 #define QGSCAMERACONTROLLER_H
 
 #include "qgis_3d.h"
+#include "qgsoptional.h"
 
 #include <Qt3DCore/QEntity>
 #include <Qt3DInput/QMouseEvent>
@@ -326,7 +327,7 @@ class _3D_EXPORT QgsCameraController : public QObject
      * Returns the minimum depth value in the square [px - 3, px + 3] * [py - 3, py + 3]
      * If the value is 1, the average depth of all non void pixels is returned instead.
      */
-    double sampleDepthBuffer( const QImage &buffer, int px, int py );
+    double sampleDepthBuffer( int px, int py );
 
 #ifndef SIP_RUN
     //! Converts screen point to world position
@@ -350,6 +351,7 @@ class _3D_EXPORT QgsCameraController : public QObject
 
     bool mDepthBufferIsReady = false;
     QImage mDepthBufferImage;
+    QgsOptional<double> mDepthBufferNonVoidAverage;
 
     std::unique_ptr<Qt3DRender::QCamera> mCameraBefore;