Add youtube video parsing

* Add saving `youtube` mode to `app_reporting_pack.yaml`
* Document using the `youtube` mode of fetching video orientation
* Add option of specifying how to save the config
* Add `--generate-config-only` to stop script execution when config is
  generated

Change-Id: I6fd672d46422d9e0e7c8bf891eecaf5b192b5a84

docs/setup-youtube-api-to-fetch-video-orientation.md

@@ -0,0 +1,36 @@
+# Setup YouTube Data API to fetch video orientation
+
+You can fetch video orientation from YouTube Data API.
+This data only available to that video's owner.
+
+In order to get access to video orientation details you need to be properly authorized.
+
+1. Create file `youtube_config.yaml` with the following elements.
+
+```
+client_id:
+client_secret:
+refresh_token:
+```
+
+2. Setup Google Cloud project and OAuth client id
+
+Create an OAuth credentials (for the first time you'll need to create a concent screen either).
+Please note you need to generate OAuth2 credentials for **desktop application**.
+Copy `client_id` and `client_secret` to `youtube_config.yaml`.
+
+
+First you need to download [oauth2l](https://github.com/google/oauth2l) tool ("oauth tool").
+For Windows and Linux please download [pre-compiled binaries](https://github.com/google/oauth2l#pre-compiled-binaries),
+for MacOS you can install via Homebrew: `brew install oauth2l`.
+
+As soon as you generated OAuth2 credentials:
+* Click the download icon next to the credentials that you just created and save file to your computer
+* Copy the file name under which you saved secrets file -
+`~/client_secret_XXX.apps.googleusercontent.com.json` where XXX will be values specific to your project
+(or just save it under `client_secret.json` name for simplicity)
+* Run desktop authentication with downloaded credentials file using oauth2l in the same folder where you put the downloaded secret file (assuming its name is client_secret.json):
+```
+oauth2l fetch --credentials ./client_secret.json  --scope youtube --output_format refresh_token
+```
+* Copy a refresh token from the output and add to `youtube_config.yaml`

run-local.sh

@@ -30,13 +30,15 @@ Helper script for running App Reporting Pack queries.\n\n
 --legacy - generates legacy views that can be plugin into existing legacy dashboard.\n
 --backfill - whether to perform backfill of the bid and budgets snapshots.\n
 --backfill-only - perform only backfill of the bid and budgets snapshots.\n
+--generate-config-only - perform only config generation instead of fetching data from Ads API.\n
 "
 
 solution_name="App Reporting Pack"
 solution_name_lowercase=$(echo $solution_name | tr '[:upper:]' '[:lower:]' |\
   tr ' ' '_')
 
 quiet="n"
+generate_config_only="n"
 
 while :; do
 case $1 in
@@ -64,6 +66,9 @@ case $1 in
   --backfill-only)
     backfill_only="y"
     ;;
+  --generate-config-only)
+    generate_config_only="y"
+    ;;
   -h|--help)
     echo -e $usage;
     exit
@@ -102,10 +107,21 @@ setup() {
   read -r save_config_answer
   save_config_answer=$(convert_answer $save_config_answer)
   if [[ $save_config_answer = "y" ]]; then
+    echo -n "Config will be saved to $solution_name_lowercase.yaml, do you want to save it here? Continue[Y] or Change[n]: "
+    read -r config_change_answer
+    config_change_answer=$(convert_answer $config_change_answer)
+    if [[ $config_change_answer = "n" ]]; then
+      echo -n "Enter name of the config (without .yaml file extension): "
+      read -r solution_name_lowercase
+    fi
     save_config="--save-config --config-destination=$solution_name_lowercase.yaml"
     echo -e "${COLOR}Saving configuration to $solution_name_lowercase.yaml${NC}"
     fetch_reports $save_config --log=$loglevel --api-version=$API_VERSION --dry-run
     generate_output_tables $save_config --log=$loglevel --dry-run
+    fetch_video_orientation $save_config --log=$loglevel --dry-run
+    if [[ $generate_config_only = "y" ]]; then
+      exit 1
+    fi
   elif [[ $save_config_answer = "q" ]]; then
     exit 1
   fi

scripts/fetch_video_orientation.py

@@ -12,14 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple, Union
 from dataclasses import dataclass, asdict
 import argparse
 import os
 import logging
 import google
 from rich.logging import RichHandler
 import yaml
+from smart_open import open
 from googleapiclient.discovery import build
 
 from gaarf.api_clients import GoogleAdsApiClient
@@ -48,9 +49,10 @@ class VideoOrientationRegexp:
 
 
 def update_config(
-        path: str,
-        mode: str,
-        video_orientation_regexp: Optional[VideoOrientationRegexp] = None):
+    path: str,
+    mode: str,
+    youtube_config_path: str = None,
+    video_orientation_regexp: Optional[VideoOrientationRegexp] = None):
     if os.path.exists(path):
         with open(path, "r", encoding="utf-8") as f:
             config = yaml.safe_load(f)
@@ -60,6 +62,9 @@ def update_config(
     if video_orientation_regexp:
         scripts_config["scripts"]["video_orientation"].update(
             asdict(video_orientation_regexp))
+    if youtube_config_path:
+        scripts_config["scripts"]["video_orientation"].update(
+            {"youtube_config_path": youtube_config_path})
     config.update(scripts_config)
     with open(path, "w", encoding="utf-8") as f:
         yaml.dump(config,
@@ -166,7 +171,12 @@ def main():
     parser.add_argument("--orientation-delimiter",
                         dest="orientation_delimiter",
                         default=None)
+    parser.add_argument("--youtube-config-path",
+                        dest="youtube_config_path",
+                        default=None)
+    parser.add_argument("--dry-run", dest="dry_run", action="store_true")
     parser.set_defaults(save_config=False)
+    parser.set_defaults(dry_run=False)
     args = parser.parse_known_args()
 
     logging.basicConfig(format="%(message)s",
@@ -183,23 +193,39 @@ def main():
         with open(args[0].gaarf_config, "r", encoding="utf-8") as f:
             gaarf_config = yaml.safe_load(f)
             if scripts := gaarf_config.get("scripts"):
-                video_orientation_config = scripts.get("video_orientation")
-                mode = video_orientation_config.get("mode")
-                element_delimiter = video_orientation_config.get(
-                    "element_delimiter")
-                orientation_position = video_orientation_config.get(
-                    "orientation_position")
-                orientation_delimiter = video_orientation_config.get(
-                    "orientation_delimiter")
+                if video_orientation_config := scripts.get(
+                        "video_orientation"):
+                    mode = video_orientation_config.get("mode")
+                    element_delimiter = video_orientation_config.get(
+                        "element_delimiter")
+                    orientation_position = video_orientation_config.get(
+                        "orientation_position")
+                    orientation_delimiter = video_orientation_config.get(
+                        "orientation_delimiter")
+                    youtube_config_path = video_orientation_config.get(
+                        "youtube_config_path")
             else:
                 element_delimiter = None
                 orientation_position = None
                 orientation_delimiter = None
 
+    save_config = args[0].save_config
+    dry_run = args[0].dry_run
+    if dry_run:
+        save_config = True
     bq_config = GaarfBqConfigBuilder(args).build()
     bq_executor = BigQueryExecutor(bq_config.project)
     if mode == "youtube":
+        if save_config:
+            update_config(path=args[0].gaarf_config,
+                          mode=mode,
+                          youtube_config_path=args[0].youtube_config_path)
+
+        if dry_run:
+            exit()
         logger.info("Getting video orientation from YouTube")
+        with open(args[0].youtube_config_path or youtube_config_path, "r", encoding="utf-8") as f:
+            youtube_config = yaml.safe_load(f)
         try:
             parsed_videos = bq_executor.execute(script_name="existing_videos",
                                                 query_text="""
@@ -229,10 +255,10 @@ def main():
         if videos:
             credentials = google.oauth2.credentials.Credentials(
                 None,
-                refresh_token=google_ads_config_dict.get("yt_refresh_token"),
+                refresh_token=youtube_config.get("refresh_token"),
                 token_uri="https://oauth2.googleapis.com/token",
-                client_id=google_ads_config_dict.get("client_id"),
-                client_secret=google_ads_config_dict.get("client_secret"))
+                client_id=youtube_config.get("client_id"),
+                client_secret=youtube_config.get("client_secret"))
             youtube_data_connector = YouTubeDataConnector(credentials)
             try:
                 video_orientations = youtube_data_connector.get_response(
@@ -250,36 +276,43 @@ def main():
             logger.info("No new videos to parse")
 
     elif mode == "regex":
-        logger.info("Parsing video orientation from asset name based on regexp")
+        logger.info(
+            "Parsing video orientation from asset name based on regexp")
         script_path = os.path.dirname(__file__)
         video_orientation_regexp = VideoOrientationRegexp(
             element_delimiter=args[0].element_delimiter,
             orientation_position=args[0].orientation_position,
             orientation_delimiter=args[0].orientation_delimiter)
+        if save_config:
+            update_config(path=args[0].gaarf_config,
+                          mode=mode,
+                          video_orientation_regexp=video_orientation_regexp)
+        if dry_run:
+            exit()
         bq_executor.execute(
             "video_orientation",
             FileReader().read(
-                os.path.join(script_path, "src/video_orientation.sql")
-            ), {
-                "macro": {
-                    "bq_dataset":
-                    bq_config.params.get("macro").get("bq_dataset"),
-                    "element_delimiter":
-                    args[0].element_delimiter or element_delimiter,
-                    "orientation_position":
-                    args[0].orientation_position or orientation_position,
-                    "orientation_delimiter":
-                    args[0].orientation_delimiter or orientation_delimiter
-                }
-            })
+                os.path.join(script_path, "src/video_orientation.sql")), {
+                    "macro": {
+                        "bq_dataset":
+                        bq_config.params.get("macro").get("bq_dataset"),
+                        "element_delimiter":
+                        args[0].element_delimiter or element_delimiter,
+                        "orientation_position":
+                        args[0].orientation_position or orientation_position,
+                        "orientation_delimiter":
+                        args[0].orientation_delimiter or orientation_delimiter
+                    }
+                })
     else:
         logger.info("Generating placeholders for video orientation")
         mode = "placeholders"
         video_orientation_regexp = None
-    if args[0].save_config:
-        update_config(path=args[0].gaarf_config,
-                      mode=mode,
-                      video_orientation_regexp=video_orientation_regexp)
+        if save_config:
+            update_config(path=args[0].gaarf_config,
+                          mode=mode)
+        if dry_run:
+            exit()
 
 
 if __name__ == "__main__":

scripts/shell_utils/app_reporting_pack.sh

@@ -50,7 +50,8 @@ ask_for_video_orientation() {
     fi
   elif [[ $video_parsing_mode = "y" ]]; then
     video_parsing_mode_output="youtube"
-    echo "Need something"
+    echo -n "Please enter path to youtube_config.yaml file: "
+    read -r youtube_config_path
 
   else
     video_parsing_mode_output="placeholders"
@@ -59,10 +60,10 @@ ask_for_video_orientation() {
 
 ask_for_cohorts() {
   default_cohorts=(0 1 3 5 7 14 30)
-  echo -n -e "${COLOR}Asset performance has cohorts for 0,1,3,5,7,14 and 30 days. Do you want to adjust it? [Y/n]: ${NC}"
+  echo -n -e "${COLOR}Asset performance has cohorts for 0,1,3,5,7,14 and 30 days. Do you want to use it? Continue[Y] or Change[n]: ${NC}"
   read -r cohorts_answer
   ads_config_answer=$(convert_answer $cohorts_answer)
-  if [[ $cohorts_answer = "y" ]]; then
+  if [[ $cohorts_answer != "y" ]]; then
     echo -n -e "${COLOR}Please enter cohort number in the following format 1,2,3,4,5: ${NC}"
     read -r cohorts_string
   fi

scripts/shell_utils/gaarf.sh

@@ -37,8 +37,10 @@ backfill_snapshots() {
 fetch_video_orientation() {
   $(which python3) $(dirname $0)/scripts/fetch_video_orientation.py \
     --mode=$video_parsing_mode_output --account=$customer_id \
-    -c=$config_file --ads-config=$ads_config \
-    --project=$project --macro.bq_dataset=$bq_dataset --element-delimiter=$element_delimiter \
+    -c=$solution_name_lowercase.yaml --ads-config=$ads_config \
+    --youtube-config-path=$youtube_config_path \
+    --project=$project --macro.bq_dataset=$bq_dataset \
+    --element-delimiter=$element_delimiter \
     --orientation-position=$orientation_position \
     --orientation-delimiter=$orientation_delimiter "$@"
 }