WIP - pluggy plugins pluggy wuggy

changedetectionio/content_fetcher.py

@@ -101,6 +101,7 @@ class Fetcher():
     error = None
     fetcher_description = "No description"
     headers = {}
+    is_plaintext = None
     instock_data = None
     instock_data_js = ""
     status_code = None

changedetectionio/flask_app.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-from changedetectionio import queuedWatchMetaData
+from changedetectionio import queuedWatchMetaData, html_tools, __version__
 from copy import deepcopy
 from distutils.util import strtobool
 from feedgen.feed import FeedGenerator
@@ -34,8 +34,6 @@
 )
 
 from flask_paginate import Pagination, get_page_parameter
-
-from changedetectionio import html_tools, __version__
 from changedetectionio.api import api_v1
 
 datastore = None
@@ -49,6 +47,18 @@
 update_q = queue.PriorityQueue()
 notification_q = queue.Queue()
 
+
+def get_plugin_manager():
+    import pluggy
+    from changedetectionio.plugins import hookspecs
+    from changedetectionio.plugins import whois as whois_plugin
+
+    pm = pluggy.PluginManager("eggsample")
+    pm.add_hookspecs(hookspecs)
+    pm.load_setuptools_entrypoints("eggsample")
+    pm.register(whois_plugin)
+    return pm
+
 app = Flask(__name__,
             static_url_path="",
             static_folder="static",
@@ -95,7 +105,6 @@ def init_app_secret(datastore_path):
 
     return secret
 
-
 @app.template_global()
 def get_darkmode_state():
     css_dark_mode = request.cookies.get('css_dark_mode', 'false')
@@ -626,7 +635,6 @@ def edit_page(uuid):
             form.fetch_backend.choices.append(p)
 
         form.fetch_backend.choices.append(("system", 'System settings default'))
-
         # form.browser_steps[0] can be assumed that we 'goto url' first
 
         if datastore.proxy_list is None:
@@ -727,6 +735,8 @@ def edit_page(uuid):
             if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
                 is_html_webdriver = True
 
+            processor_config = next((p[2] for p in processors.available_processors() if p[0] == watch.get('processor')), None)
+
             # Only works reliably with Playwright
             visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver
             output = render_template("edit.html",
@@ -741,6 +751,7 @@ def edit_page(uuid):
                                      is_html_webdriver=is_html_webdriver,
                                      jq_support=jq_support,
                                      playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),
+                                     processor_config=processor_config,
                                      settings_application=datastore.data['settings']['application'],
                                      using_global_webdriver_wait=default['webdriver_delay'] is None,
                                      uuid=uuid,

changedetectionio/forms.py

@@ -410,7 +410,7 @@ class quickWatchForm(Form):
     url = fields.URLField('URL', validators=[validateURL()])
     tags = StringTagUUID('Group tag', [validators.Optional()])
     watch_submit_button = SubmitField('Watch', render_kw={"class": "pure-button pure-button-primary"})
-    processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff")
+    processor = RadioField(u'Processor', choices=[t[:2] for t in processors.available_processors()], default="text_json_diff")
     edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
 
 
@@ -427,7 +427,7 @@ class commonSettingsForm(Form):
                                                                                                                                     message="Should contain one or more seconds")])
 class importForm(Form):
     from . import processors
-    processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff")
+    processor = RadioField(u'Processor', choices=[t[:2] for t in processors.available_processors()], default="text_json_diff")
     urls = TextAreaField('URLs')
     xlsx_file = FileField('Upload .xlsx file', validators=[FileAllowed(['xlsx'], 'Must be .xlsx file!')])
     file_mapping = SelectField('File mapping', [validators.DataRequired()], choices={('wachete', 'Wachete mapping'), ('custom','Custom mapping')})

changedetectionio/plugins/__init__.py

@@ -0,0 +1,6 @@
+import pluggy
+
+hookimpl = pluggy.HookimplMarker("eggsample")
+"""Marker to be imported and used in plugins (and for own implementations)"""
+
+x=1

changedetectionio/plugins/hookspecs.py

@@ -0,0 +1,20 @@
+import pluggy
+from changedetectionio.store import ChangeDetectionStore
+
+hookspec = pluggy.HookspecMarker("eggsample")
+
+
+@hookspec
+def extra_processor():
+    """Defines a new fetch method
+
+    :return: a tuples, (machine_name, description)
+    """
+
+@hookspec(firstresult=True)
+def processor_call(processor_name: str, datastore: ChangeDetectionStore, watch_uuid: str):
+    """
+    Call processors with processor name
+    :param processor_name: as defined in extra_processors
+    :return: data?
+    """

changedetectionio/plugins/whois.py

@@ -0,0 +1,49 @@
+"""
+Whois information lookup
+- Fetches using whois
+- Extends the 'text_json_diff' so that text filters can still be used with whois information
+"""
+
+from ..plugins import hookimpl
+import changedetectionio.processors.text_json_diff as text_json_diff
+from changedetectionio import content_fetcher
+
+# would be changedetectionio.plugins in other apps
+
+class text_json_filtering_whois(text_json_diff.perform_site_check):
+
+    def __init__(self, *args, datastore, watch_uuid, **kwargs):
+        super().__init__(*args, datastore=datastore, watch_uuid=watch_uuid, **kwargs)
+
+    def call_browser(self):
+        import whois
+        # the whois data
+        self.fetcher = content_fetcher.Fetcher()
+        self.fetcher.is_plaintext = True
+
+        from urllib.parse import urlparse
+        parsed = urlparse(self.watch.link)
+        w = whois.whois(parsed.hostname)
+        self.fetcher.content= w.text
+
+
+@hookimpl
+def extra_processor():
+    from changedetectionio.processors import default_processor_config
+    processor_config = dict(default_processor_config)
+    # Which UI elements are not used
+    processor_config['needs_request_fetch_method'] = False
+    processor_config['needs_browsersteps'] = False
+    processor_config['needs_visualselector'] = False
+    return ('plugin_processor_whois', "Whois domain information fetch", processor_config)
+
+@hookimpl
+def processor_call(processor_name, datastore, watch_uuid):
+    if processor_name == 'plugin_processor_whois':
+        x = text_json_filtering_whois(datastore=datastore, watch_uuid=watch_uuid)
+        return x
+    return None
+
+@hookimpl
+def eggsample_prep_condiments(condiments):
+    condiments["mint sauce"] = 1

changedetectionio/processors/__init__.py

@@ -6,6 +6,15 @@
 from copy import deepcopy
 from distutils.util import strtobool
 
+# Which UI elements in settings the processor requires
+# For example, restock monitor isnt compatible with visualselector and filters
+default_processor_config = {
+    'needs_request_fetch_method': True,
+    'needs_browsersteps': True,
+    'needs_visualselector': True,
+    'needs_filters': True,
+}
+
 class difference_detection_processor():
 
     browser_steps = None
@@ -131,6 +140,15 @@ def run_changedetection(self, uuid, skip_when_checksum_same=True):
 
 def available_processors():
     from . import restock_diff, text_json_diff
-    x=[('text_json_diff', text_json_diff.name), ('restock_diff', restock_diff.name)]
-    # @todo Make this smarter with introspection of sorts.
+    from ..flask_app import get_plugin_manager
+    pm = get_plugin_manager()
+    x = [('text_json_diff', text_json_diff.name, dict(default_processor_config)),
+         ('restock_diff', restock_diff.name, dict(default_processor_config))
+         ]
+
+    plugin_choices = pm.hook.extra_processor()
+    if plugin_choices:
+        for p in plugin_choices:
+            x.append(p)
+
     return x

changedetectionio/processors/text_json_diff.py

@@ -155,7 +155,7 @@ def run_changedetection(self, uuid, skip_when_checksum_same=True):
             html_content = self.fetcher.content
 
             # If not JSON,  and if it's not text/plain..
-            if 'text/plain' in self.fetcher.get_all_headers().get('content-type', '').lower():
+            if 'text/plain' in self.fetcher.get_all_headers().get('content-type', '').lower() or self.fetcher.is_plaintext:
                 # Don't run get_text or xpath/css filters on plaintext
                 stripped_text_from_html = html_content
             else:

changedetectionio/templates/edit.html

@@ -39,12 +39,15 @@
         <ul>
             <li class="tab" id=""><a href="#general">General</a></li>
             <li class="tab"><a href="#request">Request</a></li>
-            {% if playwright_enabled %}
+            {% if playwright_enabled and processor_config['needs_browsersteps'] %}
             <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
             {% endif %}
 
-            {% if watch['processor'] == 'text_json_diff' %}
+            {% if processor_config['needs_visualselector']  %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
+            {% endif %}
+
+            {% if processor_config['needs_filters']  %}
             <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
             {% endif %}
 
@@ -67,16 +70,12 @@
                         {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
                         <span class="pure-form-message-inline">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></span><br>
                         <span class="pure-form-message-inline">You can use variables in the URL, perfect for inserting the current date and other logic, <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a></span><br>
-                        <span class="pure-form-message-inline">
-                        {% if watch['processor'] == 'text_json_diff' %}
-                            Current mode: <strong>Webpage Text/HTML, JSON and PDF changes.</strong><br>
-                          <a href="{{url_for('edit_page', uuid=uuid)}}?switch_processor=restock_diff" class="pure-button button-xsmall">Switch to re-stock detection mode.</a>
-                        {% else %}
-                        Current mode: <strong>Re-stock detection.</strong><br>
-                          <a href="{{url_for('edit_page', uuid=uuid)}}?switch_processor=text_json_diff" class="pure-button button-xsmall">Switch to Webpage Text/HTML, JSON and PDF changes mode.</a>
-                        {% endif %}
-                        </span>
-
+                    </div>
+                <div class="pure-control-group">
+                    <label for="title">Processing mode</label>
+                          {% for a in available_processors %}
+                            <a href="{{url_for('edit_page', uuid=uuid)}}?switch_processor={{ a[0] }}" class="pure-button button-xsmall {% if watch['processor'] == a[0] %}button-secondary{% endif %}">{{ a[1]}}.</a>
+                          {% endfor %}
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.title, class="m-d") }}
@@ -193,7 +192,7 @@
                     </div>
             </fieldset>
             </div>
-            {% if playwright_enabled %}
+            {% if playwright_enabled and processor_config['needs_browsersteps'] %}
             <div class="tab-pane-inner" id="browser-steps">
                 <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
                 <fieldset>
@@ -264,7 +263,7 @@ <h2 >Click here to Start</h2>
                 </fieldset>
             </div>
 
-            {% if watch['processor'] == 'text_json_diff' %}
+            {% if processor_config['needs_filters']  %}
             <div class="tab-pane-inner" id="filters-and-triggers">
                     <div class="pure-control-group">
                             <strong>Pro-tips:</strong><br>
@@ -423,7 +422,7 @@ <h3>Text filtering</h3>
             </div>
             {% endif %}
 
-            {% if watch['processor'] == 'text_json_diff' %}
+            {% if processor_config['needs_visualselector']  %}
             <div class="tab-pane-inner visual-selector-ui" id="visualselector">
                 <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
 

changedetectionio/update_worker.py

@@ -259,6 +259,13 @@ def run(self):
                             update_handler = restock_diff.perform_site_check(datastore=self.datastore,
                                                                              watch_uuid=uuid
                                                                              )
+                        elif processor.startswith('plugin_processor_'):
+                            from .flask_app import get_plugin_manager
+                            pm = get_plugin_manager()
+                            x = pm.hook.processor_call(processor_name=processor, datastore=self.datastore, watch_uuid=uuid)
+                            if x:
+                                update_handler = x
+
                         else:
                             # Used as a default and also by some tests
                             update_handler = text_json_diff.perform_site_check(datastore=self.datastore,

requirements.txt

@@ -72,3 +72,5 @@ pytest-flask ~=1.2
 
 # Pin jsonschema version to prevent build errors on armv6 while rpds-py wheels aren't available (1708)
 jsonschema==4.17.3
+
+pluggy