[SPARK-32620][SQL] Reset the numPartitions metric when DPP is enabled

### What changes were proposed in this pull request?

This pr reset the `numPartitions` metric when DPP is enabled. Otherwise, it is always a [static value](https://github.com/apache/spark/blob/18cac6a9f0bf4a6d449393f1ee84004623b3c893/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala#L215).

### Why are the changes needed?

Fix metric issue.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Unit test and manual test

For [this test case](https://github.com/apache/spark/blob/18cac6a9f0bf4a6d449393f1ee84004623b3c893/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala#L252-L280).

Before this pr:
![image](https://user-images.githubusercontent.com/5399861/90301798-9310b480-ded4-11ea-9294-49bcaba46f83.png)

After this pr:
![image](https://user-images.githubusercontent.com/5399861/90301709-0fef5e80-ded4-11ea-942d-4d45d1dd15bc.png)

Closes #29436 from wangyum/SPARK-32620.

Authored-by: Yuming Wang <[email protected]>
Signed-off-by: Yuming Wang <[email protected]>
(cherry picked from commit 1354cf0)
Signed-off-by: Yuming Wang <[email protected]>

Author: wangyum

sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala

@@ -239,6 +239,7 @@ case class FileSourceScanExec(
       setFilesNumAndSizeMetric(ret, false)
       val timeTakenMs = (System.nanoTime() - startTime) / 1000 / 1000
       driverMetrics("pruningTime") = timeTakenMs
+      driverMetrics("numPartitions") = ret.length
       ret
     } else {
       selectedPartitions

sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala

@@ -319,6 +319,8 @@ abstract class DynamicPartitionPruningSuiteBase
       SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
       SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       withTable("fact", "dim") {
+        val numPartitions = 10
+
         spark.range(10)
           .map { x => Tuple3(x, x + 1, 0) }
           .toDF("did", "d1", "d2")
@@ -328,7 +330,7 @@ abstract class DynamicPartitionPruningSuiteBase
           .saveAsTable("dim")
 
         spark.range(100)
-          .map { x => Tuple2(x, x % 10) }
+          .map { x => Tuple2(x, x % numPartitions) }
           .toDF("f1", "fid")
           .write.partitionBy("fid")
           .format(tableFormat)
@@ -355,6 +357,7 @@ abstract class DynamicPartitionPruningSuiteBase
         assert(!scan1.metrics.contains("staticFilesSize"))
         val allFilesNum = scan1.metrics("numFiles").value
         val allFilesSize = scan1.metrics("filesSize").value
+        assert(scan1.metrics("numPartitions").value === numPartitions)
 
         // No dynamic partition pruning, so no static metrics
         // Only files from fid = 5 partition are scanned
@@ -367,6 +370,7 @@ abstract class DynamicPartitionPruningSuiteBase
         val partFilesSize = scan2.metrics("filesSize").value
         assert(0 < partFilesNum && partFilesNum < allFilesNum)
         assert(0 < partFilesSize && partFilesSize < allFilesSize)
+        assert(scan2.metrics("numPartitions").value === 1)
 
         // Dynamic partition pruning is used
         // Static metrics are as-if reading the whole fact table
@@ -378,6 +382,7 @@ abstract class DynamicPartitionPruningSuiteBase
         assert(scan3.metrics("staticFilesSize").value == allFilesSize)
         assert(scan3.metrics("numFiles").value == partFilesNum)
         assert(scan3.metrics("filesSize").value == partFilesSize)
+        assert(scan3.metrics("numPartitions").value === 1)
       }
     }
   }