Prevent rate limitting from ponytube, other smol adjustments

Brambles-c · Brambles-c · commit 686e26ba4d40 · 2024-10-14T08:48:22.000-07:00
diff --git a/.env.txt b/.env.txt
@@ -27,11 +27,17 @@ CHECK_BLACKLISTED=false
 # If the playlist is missing a url from the pony archive, this setting
 # will tell the program what to do
 # This setting can be set to:
-# true : The missing url will be automatically added
-# ask  : You will be asked whether or not the video should be added
-#        each time a missing video is found
-# any other input : No videos will be added to the playlist
+# true :
+#   The missing url will be automatically added
+# ask (RECOMMENDED) : You will be asked whether or not the video should be added
+#   each time a missing video is found. Recomended since some videos manually added
+#   to the playlist aren't in the archive *
+# any other input :
+#   No videos will be added to the playlist
 ADD_IF_MISSING=ask
+# * Case [10/13/24] : Don't Mine at Night (re-upload) by Jan Animation Studios has a
+# link in the playlist, but not in the archive. Replacement of the alt link was likely forgotten
+# so this would be a case where you'd want the program not to enter the alt link
 
 # If this is set to true, then any video found in the cytube playlist that
 # has a bad status label in the archive will be reported at the end of the
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,3 @@
-package-lock.json
 .env
-node_modules
+node_modules
+.vscode
diff --git a/README.md b/README.md
@@ -5,7 +5,10 @@ Written in js because:
 2. This project uses [Puppeteer](https://github.com/puppeteer/puppeteer) and a still functional [plugin](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth) that enables automation without being stopped by Cloudflare
 3. The python [alternative](https://github.com/MeiK2333/pyppeteer_stealth) is pretty outdated and was transplanted by the Puppeteer plugin anyway
 
-Using this requires [Node.js](https://nodejs.org/en) and a list of dependencies is listed in [package.json](https://github.com/Brambles-cat/ArchiveToCytube/blob/main/package.json)
+Using this requires [Node.js](https://nodejs.org/en) and a list of dependencies is listed in [package.json](https://github.com/Brambles-cat/ArchiveToCytube/blob/main/package.json) which can be installed by running
+```bash
+npm i
+```
 
 ## Usage
 Firstly, the `CHANNEL` variable in .env must be set to the name of the Cytube channel the program should run using. Several settings in this file can also be modified to change how the program behaves. Then in the command prompt, navigate to the same directory as index.js and run
diff --git a/cytube_auto.js b/cytube_auto.js
@@ -1,7 +1,7 @@
 const puppeteer = require('puppeteer-extra')
 const StealthPlugin = require('puppeteer-extra-plugin-stealth')
 
-const { csv_map: index, blacklisted_creator, get_row_video_ids, get_archive_csv, log, logErr, delay, getInput } = require('./utils.js')
+const { csv_map: index, blacklisted_creator, get_row_links, get_archive_csv, log, logErr, delay, getInput, vid_identifier } = require('./utils.js')
 
 puppeteer.use(StealthPlugin())
 
@@ -24,7 +24,7 @@ function update_playlist(use_cookie, headless, queue_delay, playlist_url, check_
         
         if (has_edit_permissions) {
             try {
-                await delay(1000)
+                await delay(2000)
                 // This is the + button which is needed to reveal the playlist adding video options
                 await page.waitForSelector('#showmediaurl').then(button => button.click())
                 await delay(1000)
@@ -69,7 +69,7 @@ function update_playlist(use_cookie, headless, queue_delay, playlist_url, check_
             row_is_blacklisted,
             csv_row = 1,
             add_vid_attempts = 0,
-            row_video_ids,
+            row_links,
             included,
             should_queue = true,
             is_contradictory = false
@@ -85,8 +85,8 @@ function update_playlist(use_cookie, headless, queue_delay, playlist_url, check_
         for (const archive_row of archive_data) {
             ++csv_row
             row_is_blacklisted = check_blacklisted && blacklisted_creator(archive_row)
-            row_video_ids = await get_row_video_ids(archive_row)
-            included = check_includes(playlistSnapshot, row_video_ids)
+            row_links = get_row_links(archive_row)
+            included = await check_includes(playlistSnapshot, row_links)
 
             if (row_is_blacklisted) {
                 if (included.video_id) {
@@ -138,31 +138,31 @@ function update_playlist(use_cookie, headless, queue_delay, playlist_url, check_
             if (archive_row[index.NOTES].includes("age restriction")) {
                 log(`${csv_row}: not present - ${archive_row[index.TITLE]}`)
 
-                if (should_queue = can_add(ask_before_adding)) {
+                if (should_queue = can_update(ask_before_adding)) {
                     log("adding using age-restriction bypassing link...\n")
                     await page.type('#mediaurl', archive_row[index.NOTES].split(" ").at(-1))
                 }
             }
             else if (!archive_row[index.STATE]) {
                 log(`${csv_row}: not present - ${archive_row[index.TITLE]}`)
 
-                if (should_queue = can_add(ask_before_adding)) {
+                if (should_queue = can_update(ask_before_adding)) {
                     log("adding...\n")
                     await page.type('#mediaurl', archive_row[index.LINK])
                 }
             }
             else if (archive_row[index.FOUND] === "found") {
                 log(`${csv_row}: not present - ${archive_row[index.TITLE]}`)
 
-                if (should_queue = can_add(ask_before_adding)) {
+                if (should_queue = can_update(ask_before_adding)) {
                     log("adding using alt link...\n")
                     await page.type('#mediaurl', archive_row[index.ALT_LINK])
                 }
             }
-            else if (row_video_ids.length === 3) {
+            else if (row_links.length === 3) {
                 log(`${csv_row}: not present - ${archive_row[index.TITLE]}`)
 
-                if (should_queue = can_add(ask_before_adding)) {
+                if (should_queue = can_update(ask_before_adding)) {
                     log("adding using link in notes...\n")
                     await page.type('#mediaurl', archive_row[index.NOTES].split(" ").at(-1))
                 }
@@ -282,14 +282,20 @@ function update_playlist(use_cookie, headless, queue_delay, playlist_url, check_
                     else if (archive_row[index.FOUND] === "found")
                         url_to_add = archive_row[index.ALT_LINK]
                     else if (archive_row[index.NOTES].includes("://"))
-                        // precondition: all urls in notes are at the end
+                        // precondition: all urls in notes are at the end of the cell
                         url_to_add = archive_row[index.NOTES].split(" ").at(-1)
                     else {
                         logErr("No useable links found", false)
+                        if (can_update(true, "Remove bad url only?")) {
+                            if (await page.$('.server-msg-disconnect')) {
+                                logErr('Disconnected from server because of duplicate login')
+                                return
+                            }
+                        }
                         continue
                     }
 
-                    if (can_add(true, "Remove bad url and replace with an alternative?")) {
+                    if (can_update(true, "Remove bad url and replace with an alternative?")) {
                         if (await page.$('.server-msg-disconnect')) {
                             logErr('Disconnected from server because of duplicate login')
                             return
@@ -372,12 +378,12 @@ async function normal_login(page, url, headless) {
 
 id_indices = [index.LINK, index.ALT_LINK, index.NOTES]
 
-function check_includes(playlist_snapshot, video_ids) {
+async function check_includes(playlist_snapshot, video_links) {
     const ret = {archive_index: null, video_id: null}
     let id, i = 0
 
-    for (;i < video_ids.length; ++i) {
-        id = video_ids[i]
+    for (;i < video_links.length; ++i) {
+        id = await vid_identifier(video_links[i])
 
         if (typeof id === "string") {
             if (id in playlist_snapshot) {
@@ -402,7 +408,7 @@ function check_includes(playlist_snapshot, video_ids) {
     return ret    
 }
 
-function can_add(ask=true, msg="Add this video to the playlist?") {
+function can_update(ask=true, msg="Add this video to the playlist?") {
     return (
         has_edit_permissions &&
         (
diff --git a/index.js b/index.js
@@ -3,14 +3,14 @@ const { logErr, log, setErrDelay, blacklist_check } = require('./utils.js')
 require('dotenv').config()
 
 
-function get(v) {
-  if (v === undefined || v.toLowerCase() === "false")
+function get(variable) {
+  if (variable === undefined || variable.toLowerCase() === "false")
     return false
 
-  if (v.toLowerCase() === "true")
+  if (variable.toLowerCase() === "true")
     return true
 
-  throw new Error(`\"${v}\" used for boolean variable. Expected \"true\" or \"false\"`)
+  throw new Error(`\"${variable}\" used for boolean variable. Expected \"true\" or \"false\"`)
 }
 
 function int(v) {
diff --git a/package-lock.json b/package-lock.json
diff --git a/utils.js b/utils.js
