improve accept language header

Author: nleush

config.js

@@ -55,6 +55,7 @@
         FB_USER_AGENT: 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
 
         ACCEPT_LANGUAGE_SUFFIX: ';q=0.9,en;q=0.7,*;q=0.5',
+        DEFAULT_ACCEPT_LANGUAGE: 'en,*',
 
         SKIP_IFRAMELY_RENDERS: false,
         DEFAULT_ASPECT_RATIO: 16 / 9,
@@ -441,4 +442,4 @@
         config.HTTP2_RETRY_CODES[item] = 1;
     });
 
-    export default config;
+    export default config;

lib/utils.js

@@ -142,10 +142,31 @@ export function prepareRequestOptions(request_options, options) {
         }        
     } 
 
-    var lang = options && options.getProviderOptions && options.getProviderOptions('locale', 'en-US') || 'en-US';
+    const DEFAULT_LANGUAGE = 'DEFAULT';
+    var lang = options?.getProviderOptions 
+        // DEFAULT_LANGUAGE returned if value is undefined.
+        && options.getProviderOptions('locale', DEFAULT_LANGUAGE) 
+        // DEFAULT_LANGUAGE returned if value is ''.
+        || DEFAULT_LANGUAGE;
+
     if (lang && !disable_language) {
+
+        if (lang === DEFAULT_LANGUAGE) {
+            lang = CONFIG.DEFAULT_ACCEPT_LANGUAGE;
+        } else {
+
+            lang = lang.replace('_', '-');
+
+            var m = lang.match(/^(\w+)-\w+$/);
+            if (m) {
+                lang += `,${m[1]}`;
+            }
+
+            lang = lang + CONFIG.ACCEPT_LANGUAGE_SUFFIX;
+        }
+
         request_options.headers = request_options.headers || {};
-        request_options.headers['Accept-Language'] = lang.replace('_', '-') + CONFIG.ACCEPT_LANGUAGE_SUFFIX;
+        request_options.headers['Accept-Language'] = lang;
     }
 
     prepareEncodedUri(request_options, 'uri');
@@ -1388,4 +1409,4 @@ export function isBlocked(url, options, cb) {
                 return re.test(domain);
             }
         });
-}
+}