Installation via pip (#186)

Builder for https://pypi.org/project/changedetection.io/

.github/workflows/image-javascript.yml

@@ -20,6 +20,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install flake8 pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
 
       - name: Lint with flake8
         run: |
@@ -30,14 +31,14 @@ jobs:
 
       - name: Create release metadata
         run: |
-          # COPY'ed by Dockerfile into backend/ of the image, then read by the server in store.py
-          echo ${{ github.sha }} > backend/source.txt
-          echo ${{ github.ref }} > backend/tag.txt
+          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
+          echo ${{ github.sha }} > changedetectionio/source.txt
+          echo ${{ github.ref }} > changedetectionio/tag.txt
 
       - name: Test with pytest
         run: |
           # Each test is totally isolated and performs its own cleanup/reset
-          cd backend; ./run_all_tests.sh
+          cd changedetectionio; ./run_all_tests.sh
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1

.github/workflows/image-tag.yml

@@ -29,6 +29,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install flake8 pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
 
       - name: Lint with flake8
         run: |
@@ -39,14 +40,14 @@ jobs:
 
       - name: Create release metadata
         run: |
-          # COPY'ed by Dockerfile into backend/ of the image, then read by the server in store.py
-          echo ${{ github.sha }} > backend/source.txt
-          echo ${{ github.ref }} > backend/tag.txt
+          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
+          echo ${{ github.sha }} > changedetectionio/source.txt
+          echo ${{ github.ref }} > changedetectionio/tag.txt
 
       - name: Test with pytest
         run: |
           # Each test is totally isolated and performs its own cleanup/reset
-          cd backend; ./run_all_tests.sh
+          cd changedetectionio; ./run_all_tests.sh
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1

.github/workflows/image.yml

@@ -20,6 +20,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install flake8 pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
 
       - name: Lint with flake8
         run: |
@@ -30,14 +31,14 @@ jobs:
 
       - name: Create release metadata
         run: |
-          # COPY'ed by Dockerfile into backend/ of the image, then read by the server in store.py
-          echo ${{ github.sha }} > backend/source.txt
-          echo ${{ github.ref }} > backend/tag.txt
+          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
+          echo ${{ github.sha }} > changedetectionio/source.txt
+          echo ${{ github.ref }} > changedetectionio/tag.txt
 
       - name: Test with pytest
         run: |
           # Each test is totally isolated and performs its own cleanup/reset
-          cd backend; ./run_all_tests.sh
+          cd changedetectionio; ./run_all_tests.sh
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1

.github/workflows/test-only.yml

@@ -19,15 +19,28 @@ jobs:
           python -m pip install --upgrade pip
           pip install flake8 pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
       - name: Lint with flake8
         run: |
           # stop the build if there are Python syntax errors or undefined names
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
       - name: Test with pytest
         run: |
           # Each test is totally isolated and performs its own cleanup/reset
-          cd backend; ./run_all_tests.sh
+          cd changedetectionio; ./run_all_tests.sh
+
+      - name: Test that pip builds without error
+        run: |
+          pip3 --version
+          python3 -m pip install wheel
+          python3 setup.py bdist_wheel
+          python3 -m pip install dist/changedetection.io-*-none-any.whl --force
+          changedetection.io -d /tmp -p 10000 &
+          sleep 3
+          curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
+          killall -9 changedetection.io
+
 

.gitignore

@@ -5,3 +5,5 @@ datastore/url-watches.json
 datastore/*
 __pycache__
 .pytest_cache
+build
+dist

Dockerfile

@@ -47,7 +47,7 @@ COPY --from=builder /dependencies /usr/local
 ENV PYTHONPATH=/usr/local
 
 # The actual flask app
-COPY backend /app/backend
+COPY changedetectionio /app/changedetectionio
 # The eventlet server wrapper
 COPY changedetection.py /app/changedetection.py
 

MANIFEST.in

@@ -0,0 +1,4 @@
+recursive-include changedetectionio/templates *
+recursive-include changedetectionio/static *
+include changedetection.py
+global-exclude *.pyc

README-pip.md

@@ -0,0 +1,71 @@
+#  changedetection.io
+![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
+<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
+  <img src="https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io" alt="Docker Pulls"/>
+</a>
+<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
+  <img src="https://img.shields.io/github/v/release/dgtlmoon/changedetection.io" alt="Change detection latest tag version"/> 
+</a>
+
+## Self-hosted open source change monitoring of web pages.
+
+_Know when web pages change! Stay ontop of new information!_ 
+
+Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information.
+
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />
+
+#### Example use cases
+
+Know when ...
+
+- Government department updates (changes are often only on their websites)
+- Local government news (changes are often only on their websites)
+- New software releases, security advisories when you're not on their mailing list.
+- Festivals with changes
+- Realestate listing changes
+- COVID related news from government websites
+- Detect and monitor changes in JSON API responses 
+- API monitoring and alerting
+
+**Get monitoring now!**
+
+```bash
+$ pip3 install changedetection.io   
+```
+
+Specify a target for the *datastore path* with `-d` (required) and a *listening port* with `-p` (defaults to `5000`)
+
+```bash
+$ changedetection.io -d /path/to/empty/data/dir -p 5000
+```
+
+
+Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
+
+### Features
+- Website monitoring
+- Change detection of content and analyses
+- Filters on change (Select by CSS or JSON)
+- Triggers (Wait for text, wait for regex)
+- Notification support
+- JSON API Monitoring
+- Parse JSON embedded in HTML
+- (Reverse) Proxy support
+- Javascript support via WebDriver
+- RaspberriPi (arm v6/v7/64 support)
+
+See https://github.com/dgtlmoon/changedetection.io for more information.
+
+
+
+### Support us
+
+Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
+
+Please support us, even small amounts help a LOT.
+
+BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/btc-support.png" style="max-width:50%;" alt="Support us!"  />

README.md

@@ -13,6 +13,8 @@ _Know when web pages change! Stay ontop of new information!_
 
 Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information.
 
+Open source web page monitoring, notification and change detection.
+
 
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />
 
@@ -29,32 +31,34 @@ Know when ...
 - Detect and monitor changes in JSON API responses 
 - API monitoring and alerting
 
-_Need an actual Chrome runner with Javascript support? see the experimental <a href="https://github.com/dgtlmoon/changedetection.io/tree/javascript-browser">Javascript/Chrome support changedetection.io branch!</a>_
+_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver!</a>_
 
 **Get monitoring now! super simple, one command!**
 
 Run the python code on your own machine by cloning this repository, or with <a href="https://docs.docker.com/get-docker/">docker</a> and/or <a href="https://www.digitalocean.com/community/tutorial_collections/how-to-install-docker-compose">docker-compose</a>
 
-With one docker-compose command
+**Docker**
+
+With Docker composer, just clone this repository and
+```bash
+$ docker-compose up -d
+```
+Docker standalone
+```bash
+$ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
+```
 
+**Python PIP**
 ```bash
-docker-compose up -d
+$ pip3 install changedetection.io
+$ changedetection.io -d /path/to/empty/data/dir -p 5000
 ```
 
 Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
 
 _Now with per-site configurable support for using a fast built in HTTP fetcher or use a Chrome based fetcher for monitoring of JavaScript websites!_
 
-
-#### Updating to the latest version
-
-Highly recommended :)
-
-```bash
-docker pull dgtlmoon/changedetection.io
-docker-compose up -d
-```
-
+
 ### Screenshots
 
 Examining differences in content.
@@ -131,15 +135,6 @@ For more information see https://docs.python-requests.org/en/master/user/advance
 This proxy support also extends to the notifications https://github.com/caronc/apprise/issues/387#issuecomment-841718867
 
 
-### Notes
-
-- ~~Does not yet support Javascript~~
-- ~~Wont work with Cloudfare type "Please turn on javascript" protected pages~~
-- You can use the 'headers' section to monitor password protected web page changes
-
-See the experimental <a href="https://github.com/dgtlmoon/changedetection.io/tree/javascript-browser">Javascript/Chrome browser support!</a>
-
-
 ### RaspberriPi support?
 
 RaspberriPi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! 

changedetection.py

@@ -8,11 +8,11 @@
 
 import eventlet
 import eventlet.wsgi
-import backend
+import changedetectionio
 
-from backend import store
+from changedetectionio import store
 
-def main(argv):
+def main():
     ssl_mode = False
     port = os.environ.get('PORT') or 5000
     do_cleanup = False
@@ -21,7 +21,7 @@ def main(argv):
     datastore_path = os.path.join(os.getcwd(), "datastore")
 
     try:
-        opts, args = getopt.getopt(argv, "csd:p:", "port")
+        opts, args = getopt.getopt(sys.argv[1:], "csd:p:", "port")
     except getopt.GetoptError:
         print('backend.py -s SSL enable -p [port] -d [datastore path]')
         sys.exit(2)
@@ -48,8 +48,13 @@ def main(argv):
     # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
     app_config = {'datastore_path': datastore_path}
 
-    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'])
-    app = backend.changedetection_app(app_config, datastore)
+    if not os.path.isdir(app_config['datastore_path']):
+        print ("ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists.\n"
+               "Alternatively, use the -d parameter.".format(app_config['datastore_path']),file=sys.stderr)
+        sys.exit(2)
+
+    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=changedetectionio.__version__)
+    app = changedetectionio.changedetection_app(app_config, datastore)
 
     # Go into cleanup mode
     if do_cleanup:
@@ -89,4 +94,4 @@ def inject_version():
 
 
 if __name__ == '__main__':
-    main(sys.argv[1:])
+    main()