Merge pull request #2539 from spaceyuck/fandom-performer-scraper

[fandom.com] Fandom performer scraper

Author: feederbox826

scrapers/fandom-wiki.yml

@@ -0,0 +1,134 @@
+name: fandom-wiki
+performerByURL:
+  - action: scrapeXPath
+    url:
+      - fandom.com/
+    queryURL: "{url}"
+    scraper: performerScraper
+
+xPathScrapers:
+  performerScraper:
+    common:
+      $infobox: //aside[contains(@class, 'portable-infobox')]
+    performer:
+      Name: 
+        selector: $infobox/h2//text()[1]
+      Aliases:
+        selector: $infobox//div[@data-source="full_name" or @data-source="full" or @data-source="also_known_as" or @data-source="known" or @data-source="aka" or contains(@data-source,"Name") or contains(@data-source,"name") or contains(@data-source,"alias") or contains(@data-source,"Alias")]/div/text()[1]
+      Gender:
+        selector:  $infobox//div[@data-source="gender" or @data-source="Gender" or @data-source="sex"]/div//text()[1]
+      Birthdate:
+        selector: $infobox//div[contains(@data-source,"Birth") or contains(@data-source,"birth") or @data-source="born" or @data-source="dob" or @data-source="DOB"]/div//text()[1]
+        postProcess: &postprocessDate
+        # there are many formats used, many only give partial info - some examples
+        # September 26, 1981
+        # June, 2077
+        # January 7, 11942
+        # Between 1 August 1950 and 31 July 1951
+        # 2 May 1998 (aged 46/47)
+        # 1997 [3] some more text
+        - replace:
+          # remove expressions in braces (likely only giving context)
+          - regex: \([^)]*\)
+            with: ""
+          # remove commas ("September 26, 1981" -> "September 26 1981")
+          - regex: "[,]"
+            with: ""
+          # remove estimation prefixes like "ca." (might be in the middle of date string, like for an unknown exact day)
+          - regex: ^(.*\s+)?(?:c\.|ca\.)\s*(\d+.*)$
+            with: "$1 $2"
+          # remove suffixed from day ("1st" -> "1", "2nd" -> "2", "3rd" -> "3", "4th" -> "4")
+          - regex: "([0-9])(?:st|nd|rd|th)"
+            with: "$1"
+          # remove extra content around first par with format "2 May 1998" -> "May 2 1998"
+          - regex: ^(?:.*[^\d])?(\d{1,2})(?:\s+)(\w{3,})(?:\s+)(\d{4})(?:.*)?$
+            with: $2 $1 $3
+          # remove extra content around first par with format "May 2 1998" -> "May 2 1998"
+          - regex: ^(?:.*[^\w])?(\w{3,})(?:\s+)(\d{1,2})(?:\s+)(\d{4})(?:.*)?$
+            with: $1 $2 $3
+          # starts with a year -> expand to JJanuary first
+          - regex: ^(\d{4})(?:[^\d].*)?$
+            with: January 1 $1
+        - parseDate: January 2 2006
+      DeathDate:
+        selector: $infobox//div[contains(@data-source,"Death") or contains(@data-source,"death") or @data-source="died" or @data-source="dod" or @data-source="DOD"]/div//text()[1]
+        postProcess: *postprocessDate
+      Ethnicity:
+        selector: $infobox//div[@data-source="species" or contains(@data-source,"Race") or contains(@data-source,"race")]/div//text()[1]
+      Country:
+        selector: $infobox//div[@data-source="nationality"]/div//text()[1]
+      HairColor:
+        selector: $infobox//div[contains(@data-source,"hair") or contains(@data-source,"Hair")]/div//text()[1]
+      EyeColor:
+        selector: $infobox//div[contains(@data-source,"eye") or contains(@data-source,"Eye")]/div//text()[1]
+      Height:
+        selector: $infobox//div[@data-source="height"]/div//text()
+        concat: " "
+        postProcess:
+        # there are many formats used, a lot of them use multi-unit formats, or even multiple values - some examples
+        # 163 cm (5'4")[3]
+        # 5'3 (1.60m)
+        # 171 cm (5'7½")
+        # 171 cm (5'7.3"), according to source abc
+        # 5 feet 3 inches (160cm)[3]
+        # 163cm[1] (5'4")[note 1]
+        # 168 cm (with heels)
+        - replace:
+          # remove citation markers like "[3]"
+          - regex: "\\[[ ]*[0-9]+[ ]*\\]"
+            with: ""
+          # height in meters, pulls out as centimeters number ("1.63 m (5'4")" -> "163")
+          - regex: ^(?:.*[^0-9.])?([0-9]+)\.([0-9]{2}])[ ]?(?:m|M)(?:.*)?$
+            with: "$1$2"
+          # height in centimeters, pulls out just the number ("163 cm (5'4")" -> "163")
+          - regex: ^(?:.*[^0-9])?([0-9]{3})[ ]?(?:cm|CM)(?:.*)?$
+            with: "$1"
+          # pull out any three digit number from string (to catch those in cm without units)
+          - regex: ^(?:.*[^0-9])?([0-9]{3})(?:.*)?$
+            with: "$1"
+          # reduce any variant of imperial to predicatble format ("5' 4"" -> "5'4")
+          - regex: ^(?:.*[^0-9])?([0-9])\s*(?:'|ft|ft.|feet|foot)\s*([0-9])(?:.*)?$
+            with: "$1'$2"
+        - feetToCm: True
+      Weight:
+        selector: $infobox//div[@data-source="weight"]/div//text()[1]
+        postProcess:
+        # there are many formats used, a lot of them use multi-unit formats, or even multiple values - some examples
+        # 127 lb (58kg)
+        # 127 lb
+        # 58 kg (127lb)
+        # 58.2 kg (127.3 lb)
+        # 58 kg
+        - replace:
+          # in kg, pull out just the number
+          - regex: ^(?:.*[^0-9.])?([0-9]{2,3})(?:\.[0-9]+)?[ ]?(?:kg|Kg|KG)(?:.*)?$
+            with: "$1"
+          # in lb, reduce to number and unit
+          - regex: ^(?:.*[^0-9.])?([0-9]{2,3}})(?:\.[0-9]+)?[ ]?(?:lb|Lb|LB)(?:.*)?$
+            with: "$1lb"
+        # use JS to convert imperial to metric, if imperial
+        - lbToKg: true
+      Measurements:
+        # looks for one field with all values, or split fields (get combined into single string)
+        # split fields really only work corretly if the field order is still B-W-H
+        selector: $infobox//div[@data-source="measurements" or @data-source="bust" or @data-source="waist" or @data-source="hips"]/div//text()[1]
+        concat: " "
+        postProcess:
+        # there are probably many formats used - examples fo seen formats
+        # B93 W59 H88 cm (B37" W23" H35")
+        # separate fields with <number> cm
+        - replace:
+          # remove citation markers like "[3]"
+          - regex: "\\[[ ]*[0-9]+[ ]*\\]"
+            with: ""
+          # remove expressions in braces (to remove noise from potential split fields)
+          - regex: \([^)]*\)
+            with: ""
+          # reduce to B-W-H removing prefixes and units ("B93 W59 H88 cm" -> "93-59-88")
+          - regex: ^(?:.*B|.*[^0-9])?([0-9]{2,})\s*(?:cm|")?[\s\-]+W?([0-9]{2,})\s*(?:cm|")?[\s\-]+H?([0-9]{2,})\s*(?:cm|")?.*$
+            with: "$1-$2-$3"
+      Tags:
+        Name:
+          selector: //nav[@id="articleCategories"]//li[contains(@class, "category")]/@data-name
+      Image:
+        selector: $infobox//figure//img/@src