diff --git a/MANIFEST.in b/MANIFEST.in index b06768fcf5a..ce4f64dc43e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,7 @@ recursive-include changedetectionio/api * recursive-include changedetectionio/blueprint * recursive-include changedetectionio/model * +recursive-include changedetectionio/plugins * recursive-include changedetectionio/processors * recursive-include changedetectionio/res * recursive-include changedetectionio/static * diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py index 00da86c9a31..cec838604dd 100644 --- a/changedetectionio/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -101,6 +101,7 @@ class Fetcher(): error = None fetcher_description = "No description" headers = {} + is_plaintext = None instock_data = None instock_data_js = "" status_code = None diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 360eff0e0d5..6f3bed092d5 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -from changedetectionio import queuedWatchMetaData +from changedetectionio import queuedWatchMetaData, html_tools, __version__ from copy import deepcopy from distutils.util import strtobool from feedgen.feed import FeedGenerator @@ -35,8 +35,6 @@ ) from flask_paginate import Pagination, get_page_parameter - -from changedetectionio import html_tools, __version__ from changedetectionio.api import api_v1 datastore = None @@ -50,6 +48,18 @@ update_q = queue.PriorityQueue() notification_q = queue.Queue() + +def get_plugin_manager(): + import pluggy + from changedetectionio.plugins import hookspecs + from changedetectionio.plugins import whois as whois_plugin + + pm = pluggy.PluginManager("changedetectionio_plugin") + pm.add_hookspecs(hookspecs) + pm.load_setuptools_entrypoints("changedetectionio_plugin") + pm.register(whois_plugin) + return pm + app = Flask(__name__, static_url_path="", static_folder="static", @@ -96,7 +106,6 @@ def init_app_secret(datastore_path): return secret - @app.template_global() def get_darkmode_state(): css_dark_mode = request.cookies.get('css_dark_mode', 'false') @@ -629,7 +638,6 @@ def edit_page(uuid): form.fetch_backend.choices.append(p) form.fetch_backend.choices.append(("system", 'System settings default')) - # form.browser_steps[0] can be assumed that we 'goto url' first if datastore.proxy_list is None: @@ -730,6 +738,8 @@ def edit_page(uuid): if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'): is_html_webdriver = True + processor_config = next((p[2] for p in processors.available_processors() if p[0] == watch.get('processor')), None) + # Only works reliably with Playwright visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver output = render_template("edit.html", @@ -744,6 +754,7 @@ def edit_page(uuid): is_html_webdriver=is_html_webdriver, jq_support=jq_support, playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False), + processor_config=processor_config, settings_application=datastore.data['settings']['application'], using_global_webdriver_wait=default['webdriver_delay'] is None, uuid=uuid, @@ -824,11 +835,14 @@ def settings_page(): flash("An error occurred, please see below.", "error") output = render_template("settings.html", - form=form, - hide_remove_pass=os.getenv("SALTED_PASS", False), api_key=datastore.data['settings']['application'].get('api_access_token'), emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), - settings_application=datastore.data['settings']['application']) + form=form, + hide_remove_pass=os.getenv("SALTED_PASS", False), + settings_application=datastore.data['settings']['application'], + plugins=[] + + ) return output diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 9f72a748cd0..7653ad2483a 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -410,7 +410,7 @@ class quickWatchForm(Form): url = fields.URLField('URL', validators=[validateURL()]) tags = StringTagUUID('Group tag', [validators.Optional()]) watch_submit_button = SubmitField('Watch', render_kw={"class": "pure-button pure-button-primary"}) - processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff") + processor = RadioField(u'Processor', choices=[t[:2] for t in processors.available_processors()], default="text_json_diff") edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"}) @@ -427,7 +427,7 @@ class commonSettingsForm(Form): message="Should contain one or more seconds")]) class importForm(Form): from . import processors - processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff") + processor = RadioField(u'Processor', choices=[t[:2] for t in processors.available_processors()], default="text_json_diff") urls = TextAreaField('URLs') xlsx_file = FileField('Upload .xlsx file', validators=[FileAllowed(['xlsx'], 'Must be .xlsx file!')]) file_mapping = SelectField('File mapping', [validators.DataRequired()], choices={('wachete', 'Wachete mapping'), ('custom','Custom mapping')}) diff --git a/changedetectionio/model/App.py b/changedetectionio/model/App.py index 1202d5db198..36ecc8be0a6 100644 --- a/changedetectionio/model/App.py +++ b/changedetectionio/model/App.py @@ -38,6 +38,7 @@ class model(dict): 'notification_format': default_notification_format, 'notification_title': default_notification_title, 'notification_urls': [], # Apprise URL list + 'plugins': [], # list of dict, keyed by plugin name, with dict of the config and enabled true/false 'pager_size': 50, 'password': False, 'render_anchor_tag_content': False, diff --git a/changedetectionio/plugins/__init__.py b/changedetectionio/plugins/__init__.py new file mode 100644 index 00000000000..98ade07d82d --- /dev/null +++ b/changedetectionio/plugins/__init__.py @@ -0,0 +1,6 @@ +import pluggy + +hookimpl = pluggy.HookimplMarker("changedetectionio_plugin") +"""Marker to be imported and used in plugins (and for own implementations)""" + +x=1 \ No newline at end of file diff --git a/changedetectionio/plugins/hookspecs.py b/changedetectionio/plugins/hookspecs.py new file mode 100644 index 00000000000..c6d4905b664 --- /dev/null +++ b/changedetectionio/plugins/hookspecs.py @@ -0,0 +1,20 @@ +import pluggy +from changedetectionio.store import ChangeDetectionStore + +hookspec = pluggy.HookspecMarker("changedetectionio_plugin") + + +@hookspec +def extra_processor(): + """Defines a new fetch method + + :return: a tuples, (machine_name, description) + """ + +@hookspec(firstresult=True) +def processor_call(processor_name: str, datastore: ChangeDetectionStore, watch_uuid: str): + """ + Call processors with processor name + :param processor_name: as defined in extra_processors + :return: data? + """ \ No newline at end of file diff --git a/changedetectionio/plugins/whois.py b/changedetectionio/plugins/whois.py new file mode 100644 index 00000000000..96ad890467e --- /dev/null +++ b/changedetectionio/plugins/whois.py @@ -0,0 +1,53 @@ +""" +Whois information lookup +- Fetches using whois +- Extends the 'text_json_diff' so that text filters can still be used with whois information + +@todo publish to pypi and github as a separate plugin +""" + +from ..plugins import hookimpl +import changedetectionio.processors.text_json_diff as text_json_diff +from changedetectionio import content_fetcher + +# would be changedetectionio.plugins in other apps + +class text_json_filtering_whois(text_json_diff.perform_site_check): + + def __init__(self, *args, datastore, watch_uuid, **kwargs): + super().__init__(*args, datastore=datastore, watch_uuid=watch_uuid, **kwargs) + + def call_browser(self): + import whois + # the whois data + self.fetcher = content_fetcher.Fetcher() + self.fetcher.is_plaintext = True + + from urllib.parse import urlparse + parsed = urlparse(self.watch.link) + w = whois.whois(parsed.hostname) + self.fetcher.content= w.text + +@hookimpl +def extra_processor(): + """ + Advertise a new processor + :return: + """ + from changedetectionio.processors import default_processor_config + processor_config = dict(default_processor_config) + # Which UI elements are not used + processor_config['needs_request_fetch_method'] = False + processor_config['needs_browsersteps'] = False + processor_config['needs_visualselector'] = False + return ('plugin_processor_whois', "Whois domain information fetch", processor_config) + +# @todo When a watch chooses this extra_process processor, the watch should ONLY use this one. +# (one watch can only have one extra_processor) +@hookimpl +def processor_call(processor_name, datastore, watch_uuid): + if processor_name == 'plugin_processor_whois': # could be removed, see above note + x = text_json_filtering_whois(datastore=datastore, watch_uuid=watch_uuid) + return x + return None + diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py index ea8b9ffb2c8..53817bc184c 100644 --- a/changedetectionio/processors/__init__.py +++ b/changedetectionio/processors/__init__.py @@ -7,6 +7,15 @@ from distutils.util import strtobool from loguru import logger +# Which UI elements in settings the processor requires +# For example, restock monitor isnt compatible with visualselector and filters +default_processor_config = { + 'needs_request_fetch_method': True, + 'needs_browsersteps': True, + 'needs_visualselector': True, + 'needs_filters': True, +} + class difference_detection_processor(): browser_steps = None @@ -132,6 +141,15 @@ def run_changedetection(self, uuid, skip_when_checksum_same=True): def available_processors(): from . import restock_diff, text_json_diff - x=[('text_json_diff', text_json_diff.name), ('restock_diff', restock_diff.name)] - # @todo Make this smarter with introspection of sorts. + from ..flask_app import get_plugin_manager + pm = get_plugin_manager() + x = [('text_json_diff', text_json_diff.name, dict(default_processor_config)), + ('restock_diff', restock_diff.name, dict(default_processor_config)) + ] + + plugin_choices = pm.hook.extra_processor() + if plugin_choices: + for p in plugin_choices: + x.append(p) + return x diff --git a/changedetectionio/processors/text_json_diff.py b/changedetectionio/processors/text_json_diff.py index 94fea913861..338764681df 100644 --- a/changedetectionio/processors/text_json_diff.py +++ b/changedetectionio/processors/text_json_diff.py @@ -155,7 +155,7 @@ def run_changedetection(self, uuid, skip_when_checksum_same=True): html_content = self.fetcher.content # If not JSON, and if it's not text/plain.. - if 'text/plain' in self.fetcher.get_all_headers().get('content-type', '').lower(): + if 'text/plain' in self.fetcher.get_all_headers().get('content-type', '').lower() or self.fetcher.is_plaintext: # Don't run get_text or xpath/css filters on plaintext stripped_text_from_html = html_content else: diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index d43ed6665f1..868e4eaba89 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -39,12 +39,15 @@ - + +

Text filtering

@@ -423,7 +427,7 @@

Text filtering

{% endif %} - {% if watch['processor'] == 'text_json_diff' %} + {% if processor_config['needs_visualselector'] %}
diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index 508f49b2494..3be95e6d641 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -22,6 +22,7 @@
  • Global Filters
  • API
  • CAPTCHA & Proxies
    • +
  • Plugins
    • @@ -243,6 +244,12 @@ {{ render_field(form.requests.form.extra_browsers) }}
    +
    + available plugin on/off stuff here + + how to let each one expose config? +
    +
    {{ render_button(form.save_button) }} diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 538a8a329ef..e7d3301534b 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -259,6 +259,13 @@ def run(self): update_handler = restock_diff.perform_site_check(datastore=self.datastore, watch_uuid=uuid ) + elif processor.startswith('plugin_processor_'): + from .flask_app import get_plugin_manager + pm = get_plugin_manager() + x = pm.hook.processor_call(processor_name=processor, datastore=self.datastore, watch_uuid=uuid) + if x: + update_handler = x + else: # Used as a default and also by some tests update_handler = text_json_diff.perform_site_check(datastore=self.datastore, diff --git a/requirements.txt b/requirements.txt index 5bc269c9c25..e0e9dbd93ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -73,4 +73,5 @@ pytest-flask ~=1.2 # Pin jsonschema version to prevent build errors on armv6 while rpds-py wheels aren't available (1708) jsonschema==4.17.3 +pluggy loguru