Restore workflow file changes for D76242338 (#3074)

Summary:
Pull Request resolved: #3074

### Motivation
The updated workflow in Diff D76242338 has been broken due to the revert operation in Diff D76273807

### Major changes
Copy and paste workflow file configurations in D76242338 for
- `//torchrec/github/.github/workflows/unittest_ci.yml`
- `//torchrec/github/.github/workflows/unittest_ci_cpu.yml`

Reviewed By: TroyGarden

Differential Revision: D76309605

fbshipit-source-id: 95c29cd3d89272cd0558aac724e07e46a3efe012

Author: lizhouyu

.github/workflows/unittest_ci_cpu.yml

@@ -20,24 +20,33 @@ on:
 jobs:
   build_test:
     strategy:
-        fail-fast: false
-        matrix:
-          include:
-            - os: linux.2xlarge
-              python-version: 3.9
-              python-tag: "py39"
-            - os: linux.2xlarge
-              python-version: '3.10'
-              python-tag: "py310"
-            - os: linux.2xlarge
-              python-version: '3.11'
-              python-tag: "py311"
-            - os: linux.2xlarge
-              python-version: '3.12'
-              python-tag: "py312"
-            - os: linux.2xlarge
-              python-version: '3.13'
-              python-tag: "py313"
+      fail-fast: false
+      matrix:
+        os:
+          - linux.2xlarge
+        python:
+          - version: "3.9"
+            tag: "py39"
+          - version: "3.10"
+            tag: "py310"
+          - version: "3.11"
+            tag: "py311"
+          - version: "3.12"
+            tag: "py312"
+          - version: "3.13"
+            tag: "py313"
+        is_pr:
+          - ${{ github.event_name == 'pull_request' }}
+        exclude:
+          - is_pr: true
+            python:
+              version: "3.10"
+          - is_pr: true
+            python:
+              version: "3.11"
+          - is_pr: true
+            python:
+              version: "3.12"
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
     permissions:
       id-token: write