Merge pull request #2 from pyronear/rs/add-load-script

ETL Formalization

Author: MateoLostanlen

.dockerignore

@@ -0,0 +1,4 @@
+.git
+*.pyc
+__pycache__
+*.log

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+max-line-length = 120
+ignore = E203, E402, E265, F403, W503, W504, E731
+exclude = .git, venv*, docs, build
+per-file-ignores = **/__init__.py:F401

.github/collect_env.py

@@ -48,7 +48,9 @@
 
 def run(command):
     """Returns (return-code, stdout, stderr)"""
-    p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+    p = subprocess.Popen(
+        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
+    )
     output, err = p.communicate()
     rc = p.returncode
     if PY3:
@@ -99,11 +101,15 @@ def get_windows_version(run_lambda):
 
 
 def get_lsb_version(run_lambda):
-    return run_and_parse_first_match(run_lambda, "lsb_release -a", r"Description:\t(.*)")
+    return run_and_parse_first_match(
+        run_lambda, "lsb_release -a", r"Description:\t(.*)"
+    )
 
 
 def check_release_file(run_lambda):
-    return run_and_parse_first_match(run_lambda, "cat /etc/*-release", r'PRETTY_NAME="(.*)"')
+    return run_and_parse_first_match(
+        run_lambda, "cat /etc/*-release", r'PRETTY_NAME="(.*)"'
+    )
 
 
 def get_os(run_lambda):

.github/workflows/style.yml

@@ -68,11 +68,12 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -e . --upgrade
+          pip install -r requirements.txt
           pip install mypy
       - name: Run mypy
         run: |
           mypy --version
-          mypy
+          mypy --config-file pyproject.toml
 
   pydocstyle:
     runs-on: ${{ matrix.os }}

.gitignore

@@ -123,4 +123,8 @@ conda-dist/
 #project
 AWF_scrap*
 *.jpg
-*.pt
+*.pt
+
+cameras.csv
+credentials.json
+*.github*

Dockerfile

@@ -0,0 +1,13 @@
+FROM pyronear/pyro-engine:latest
+
+# set work directory
+WORKDIR /usr/src/app
+
+
+COPY requirements.txt requirements.txt
+RUN pip install --default-timeout=500 -r requirements.txt \
+    && pip cache purge \
+    && rm -rf /root/.cache/pip
+
+# Copy only the necessary application files
+COPY src/ /usr/src/app

Makefile

@@ -1,5 +1,16 @@
 # this target runs checks on all files and potentially modifies some of them
 style:
 	isort .
-	black .
+	black --diff .
 
+quality:
+	isort .
+	flake8
+	mypy
+	pydocstyle
+
+build:
+	docker build . -t pyronear/pyro-etl
+
+run-etl:
+	docker run pyronear/pyro-etl:latest  /bin/sh -c "python /app/dl_images.py && python /app/predict_load.py"

README.md

@@ -20,6 +20,8 @@ Pyro-Scrapper is a specialized tool designed for scraping images from [Alert Wil
 Certainly! Here's the revised Usage section with the added code snippets:
 
 ## Installation and Usage
+
+### Locally
 - Clone the repository.
 - Ensure you have the required dependencies by installing them from `requirements.txt`.
 
@@ -40,6 +42,16 @@ python/split_cams.py
 
 These commands will initiate the image scraping and view point splitting.
 
+
+### Using the docker image
+
+First you will need to generate the credentials.json file according to the source you want to scrapp. 
+For the alertwildfire , you can use the src/generate_wildfire_config.py script.
+
+Then you have two choices : 
+1) using the pyro-devops dev env, copying the credentials.json file in the data/ folder and launching the 'make-etl' command
+2) filling your .env locally (API_URL) in order to use an external Pyro-API , filling the token part of the credentials.json and launching the command 'make run-etl' from this repo.
+
 ## Contributing
 
 Please refer to [`CONTRIBUTING`](CONTRIBUTING.md) if you wish to contribute to this project.
@@ -51,5 +63,3 @@ This project is developed and maintained by the repo owner and volunteers from [
 ## License
 
 Distributed under the Apache 2 License. See [`LICENSE`](LICENSE) for more information.
-
-

pyproject.toml

@@ -0,0 +1,37 @@
+
+[tool.mypy]
+files = "src/"
+show_error_codes = true
+pretty = true
+warn_unused_ignores = true
+warn_redundant_casts = true
+no_implicit_optional = true
+check_untyped_defs = true
+implicit_reexport = false
+
+[[tool.mypy.overrides]]
+module = [
+    "onnxruntime.*",
+    "requests.*",
+    "PIL.*",
+    "pyroclient.*",
+    "urllib3.*",
+    "pyroengine.*",
+]
+ignore_missing_imports = true
+
+[tool.isort]
+line_length = 120
+src_paths = ["src", ".github"]
+skip_glob = "**/__init__.py"
+
+[tool.pydocstyle]
+select = "D300,D301,D417"
+match = ".*\\.py"
+
+[tool.coverage.run]
+source = ["src"]
+
+[tool.black]
+line-length = 120
+target-version = ['py38']

requirements.txt

@@ -1,8 +1,11 @@
 opencv-python
 tqdm
+types-tqdm
 requests
 python-dotenv
 pytz
+types-pytz
+Pillow
 
 #Style
 black