From 1062acf7282a542bd5aaa369ef1afbbb0ea86985 Mon Sep 17 00:00:00 2001 From: Julius <kirill.vackevich@gmail.com> Date: Mon, 6 Jul 2020 23:06:01 +0300 Subject: [PATCH 1/8] Add console interface --- arg_interface.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 arg_interface.py diff --git a/arg_interface.py b/arg_interface.py new file mode 100644 index 0000000..338b7d7 --- /dev/null +++ b/arg_interface.py @@ -0,0 +1,28 @@ +import argparse + + +def create_interface(): + parser = argparse.ArgumentParser() + parser._actions[0].help = 'Show parameters' + + parser.add_argument('-f', '--file', + help='Use your own file with urls', + action='store', + default='urls.list', + metavar='file' + ) + + parser.add_argument('-s', '--silent', + help='silent mode', + action='store_true' + ) + + parser.add_argument('-d', '--directory', + help='directory name for JSON files', + action='store', + default='.', + metavar='dir' + ) + + args = parser.parse_args() + return args From 3730f005dd93e663a6ce340f979cd781d0582fea Mon Sep 17 00:00:00 2001 From: Julius <kirill.vackevich@gmail.com> Date: Mon, 6 Jul 2020 23:07:11 +0300 Subject: [PATCH 2/8] Upload console interface in scrape.py --- scrape.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/scrape.py b/scrape.py index 32380c1..8525875 100644 --- a/scrape.py +++ b/scrape.py @@ -1,14 +1,17 @@ #!/usr/bin/env python import datetime import json +import sys +import os import youtube_dl import slugify +import arg_interface DEFAULT_LANGUAGE = "rus" -DEFAULT_SILENT_MODE = False -DEFAULT_URLS_LIST_FILENAME = "urls.list" +# DEFAULT_SILENT_MODE = False +# DEFAULT_URLS_LIST_FILENAME = "urls.list" JSON_FORMAT_KWARGS = { @@ -20,13 +23,13 @@ def main(): - list_filename = DEFAULT_URLS_LIST_FILENAME + list_filename, FOLDER_NAME = setup_interface() urls = filter(None, map(str.strip, open(list_filename).readlines())) videos_meta = filter(None, sum((get_entries(u) for u in urls), [])) for meta in videos_meta: prepared_meta = get_prepared_meta(meta) filename = generate_filename(meta) - with open(filename, "w", encoding="utf8") as json_file: + with open(FOLDER_NAME + filename, "w", encoding="utf8") as json_file: json.dump(prepared_meta, json_file, **JSON_FORMAT_KWARGS) @@ -117,5 +120,18 @@ def sanitize(title_substring): return title_substring.replace("\u200b", "").strip() +def setup_interface(): + namespace = arg_interface.create_interface() + URLS_LIST_FILENAME = namespace.file + SILENT_MODE = namespace.silent + FOLDER_NAME = namespace.directory + if not os.path.exists(FOLDER_NAME): + os.mkdir(FOLDER_NAME) + FOLDER_NAME += os.sep + if SILENT_MODE: + sys.stdout = open(os.devnull, "w") # output => null + return URLS_LIST_FILENAME, FOLDER_NAME + + if __name__ == "__main__": main() From 25fae7349763a1b407fe2b23d527c2b21276675f Mon Sep 17 00:00:00 2001 From: Julius <kirill.vackevich@gmail.com> Date: Mon, 6 Jul 2020 23:07:40 +0300 Subject: [PATCH 3/8] Update requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 5c41ca1..0132096 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ python-slugify youtube-dl +flake8 From bf13d3f3529c580c4f3bb249af8f7f0430622f1d Mon Sep 17 00:00:00 2001 From: Julius <kirill.vackevich@gmail.com> Date: Tue, 7 Jul 2020 22:07:03 +0300 Subject: [PATCH 4/8] change style of parameters --- arg_interface.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/arg_interface.py b/arg_interface.py index 338b7d7..b47e829 100644 --- a/arg_interface.py +++ b/arg_interface.py @@ -5,24 +5,27 @@ def create_interface(): parser = argparse.ArgumentParser() parser._actions[0].help = 'Show parameters' - parser.add_argument('-f', '--file', - help='Use your own file with urls', - action='store', - default='urls.list', - metavar='file' - ) + parser.add_argument( + '-f', '--file', + help='Use your own file with urls', + action='store', + default='urls.list', + metavar='file', + ) - parser.add_argument('-s', '--silent', - help='silent mode', - action='store_true' - ) + parser.add_argument( + '-s', '--silent', + help='silent mode', + action='store_true', + ) - parser.add_argument('-d', '--directory', - help='directory name for JSON files', - action='store', - default='.', - metavar='dir' - ) + parser.add_argument( + '-d', '--directory', + help='directory name for JSON files', + action='store', + default='.', + metavar='dir', + ) args = parser.parse_args() return args From 1fab675b9104da5e672118da3cfc54734daa597e Mon Sep 17 00:00:00 2001 From: Julius <kirill.vackevich@gmail.com> Date: Mon, 13 Jul 2020 23:21:28 +0300 Subject: [PATCH 5/8] separate requirements --- requirements-dev.txt | 1 + requirements.txt | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..3930480 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +flake8 diff --git a/requirements.txt b/requirements.txt index 0132096..75ee514 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,2 @@ python-slugify youtube-dl -flake8 - From 09fd48542a5a9b54654d3718acd0844d10950d8c Mon Sep 17 00:00:00 2001 From: Julius <kirill.vackevich@gmail.com> Date: Mon, 13 Jul 2020 23:23:15 +0300 Subject: [PATCH 6/8] delete silent mode --- arg_interface.py | 6 ------ scrape.py | 7 ------- 2 files changed, 13 deletions(-) diff --git a/arg_interface.py b/arg_interface.py index b47e829..3f54200 100644 --- a/arg_interface.py +++ b/arg_interface.py @@ -13,12 +13,6 @@ def create_interface(): metavar='file', ) - parser.add_argument( - '-s', '--silent', - help='silent mode', - action='store_true', - ) - parser.add_argument( '-d', '--directory', help='directory name for JSON files', diff --git a/scrape.py b/scrape.py index 8525875..4be15eb 100644 --- a/scrape.py +++ b/scrape.py @@ -10,10 +10,6 @@ import arg_interface DEFAULT_LANGUAGE = "rus" -# DEFAULT_SILENT_MODE = False -# DEFAULT_URLS_LIST_FILENAME = "urls.list" - - JSON_FORMAT_KWARGS = { "indent": 2, "separators": (",", ": "), @@ -123,13 +119,10 @@ def sanitize(title_substring): def setup_interface(): namespace = arg_interface.create_interface() URLS_LIST_FILENAME = namespace.file - SILENT_MODE = namespace.silent FOLDER_NAME = namespace.directory if not os.path.exists(FOLDER_NAME): os.mkdir(FOLDER_NAME) FOLDER_NAME += os.sep - if SILENT_MODE: - sys.stdout = open(os.devnull, "w") # output => null return URLS_LIST_FILENAME, FOLDER_NAME From 26818b397a835a594fe1bef94c9820e5b0c910a5 Mon Sep 17 00:00:00 2001 From: Julius <kirill.vackevich@gmail.com> Date: Mon, 13 Jul 2020 23:39:21 +0300 Subject: [PATCH 7/8] update return of setup function --- scrape.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scrape.py b/scrape.py index 4be15eb..c953569 100644 --- a/scrape.py +++ b/scrape.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import datetime import json -import sys import os import youtube_dl @@ -10,6 +9,7 @@ import arg_interface DEFAULT_LANGUAGE = "rus" + JSON_FORMAT_KWARGS = { "indent": 2, "separators": (",", ": "), @@ -19,7 +19,9 @@ def main(): - list_filename, FOLDER_NAME = setup_interface() + namespace = setup_interface() + list_filename = namespace.URLS_LIST_FILENAME + FOLDER_NAME = namespace.FOLDER_NAME urls = filter(None, map(str.strip, open(list_filename).readlines())) videos_meta = filter(None, sum((get_entries(u) for u in urls), [])) for meta in videos_meta: @@ -118,12 +120,11 @@ def sanitize(title_substring): def setup_interface(): namespace = arg_interface.create_interface() - URLS_LIST_FILENAME = namespace.file FOLDER_NAME = namespace.directory if not os.path.exists(FOLDER_NAME): os.mkdir(FOLDER_NAME) - FOLDER_NAME += os.sep - return URLS_LIST_FILENAME, FOLDER_NAME + namespace.FOLDER_NAME = FOLDER_NAME + os.sep + return namespace if __name__ == "__main__": From 07ee317b5f81ab0ed0720334a34bb667bfbf7e8c Mon Sep 17 00:00:00 2001 From: Julius <kirill.vackevich@gmail.com> Date: Mon, 13 Jul 2020 23:44:47 +0300 Subject: [PATCH 8/8] move argparse into scrape.py --- arg_interface.py | 25 ------------------------- scrape.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 27 deletions(-) delete mode 100644 arg_interface.py diff --git a/arg_interface.py b/arg_interface.py deleted file mode 100644 index 3f54200..0000000 --- a/arg_interface.py +++ /dev/null @@ -1,25 +0,0 @@ -import argparse - - -def create_interface(): - parser = argparse.ArgumentParser() - parser._actions[0].help = 'Show parameters' - - parser.add_argument( - '-f', '--file', - help='Use your own file with urls', - action='store', - default='urls.list', - metavar='file', - ) - - parser.add_argument( - '-d', '--directory', - help='directory name for JSON files', - action='store', - default='.', - metavar='dir', - ) - - args = parser.parse_args() - return args diff --git a/scrape.py b/scrape.py index c953569..1a804e2 100644 --- a/scrape.py +++ b/scrape.py @@ -2,11 +2,11 @@ import datetime import json import os +import argparse import youtube_dl import slugify -import arg_interface DEFAULT_LANGUAGE = "rus" @@ -118,8 +118,32 @@ def sanitize(title_substring): return title_substring.replace("\u200b", "").strip() +def create_interface(): + parser = argparse.ArgumentParser() + parser._actions[0].help = 'Show parameters' + + parser.add_argument( + '-f', '--file', + help='Use your own file with urls', + action='store', + default='urls.list', + metavar='file', + ) + + parser.add_argument( + '-d', '--directory', + help='directory name for JSON files', + action='store', + default='.', + metavar='dir', + ) + + args = parser.parse_args() + return args + + def setup_interface(): - namespace = arg_interface.create_interface() + namespace = create_interface() FOLDER_NAME = namespace.directory if not os.path.exists(FOLDER_NAME): os.mkdir(FOLDER_NAME)