Migration to github winds-mobi organization

Author: ysavary

.gitignore

@@ -0,0 +1,3 @@
+*.pyc
+/.idea/
+/local_settings.py

LICENSE.txt

@@ -0,0 +1,30 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+arrow = "==0.12.1"
+cachetools = "==1.1.3"
+lxml = "==4.0.0"
+metar = "==1.6.0"
+mysqlclient = "==1.3.6"
+numpy = "==1.12.1"
+pint = "==0.8.1"
+pyaml = "==16.9.0"
+pymongo = "==3.0.3"
+pytz = "==2018.5"
+redis = "==2.10.3"
+requests = "==2.19.1"
+sentry-sdk = "==0.6.5"
+scikit-learn = "==0.20.0"
+scipy = "==0.19.0"
+tenacity = "==5.0.2"
+xmltodict = "==0.10.2"
+
+[dev-packages]
+"flake8" = "*"
+"flake8-quotes" = "*"
+
+[requires]
+python_version = "3.6"

Pipfile

@@ -0,0 +1,24 @@
+winds.mobi - real-time weather observations
+===========================================
+
+[winds.mobi](http://winds.mobi): Paraglider pilot, kitesurfer, check real-time weather conditions of your favorite spots
+on your smartphone, your tablet or your computer.
+
+Follow this project on:
+- [Facebook](https://www.facebook.com/WindsMobi/)
+
+winds-mobi-providers
+--------------------
+
+Python 3.6 cronjobs that get the weather data from different providers and save it into mongodb. This project use
+Google Cloud APIs to compute any missing station details (altitude, name, timezone, ...). Google Cloud API results are
+cached with redis.   
+
+### How to build
+
+- pipenv install
+
+Licensing
+---------
+
+Please see the file called [LICENSE.txt](https://github.com/winds-mobi/winds-mobi-providers/blob/master/LICENSE.txt)

Pipfile.lock

@@ -0,0 +1,72 @@
+import argparse
+from datetime import datetime
+
+import numpy as np
+from commons.provider import get_logger
+from pymongo import MongoClient, uri_parser
+from scipy.spatial import KDTree
+from settings import MONGODB_URL
+from sklearn.cluster import AgglomerativeClustering
+
+log = get_logger('clusters')
+
+parser = argparse.ArgumentParser(description='Save clusters membership in mongodb')
+parser.add_argument('--num', type=int, default=50, help='Specify the number of cluster levels [default: %(default)s]')
+args = vars(parser.parse_args())
+
+uri = uri_parser.parse_uri(MONGODB_URL)
+client = MongoClient(uri['nodelist'][0][0], uri['nodelist'][0][1])
+db = client.windmobile
+
+now = datetime.now().timestamp()
+all_stations = list(db.stations.find({
+    'status': {'$ne': 'hidden'},
+    'last._id': {'$gt': now - 30 * 24 * 3600}
+}))
+range_clusters = np.geomspace(20, len(all_stations), num=args['num'], dtype=np.int)
+
+ids = np.array([station['_id'] for station in all_stations])
+
+x = [station['loc']['coordinates'][0] for station in all_stations]
+y = [station['loc']['coordinates'][1] for station in all_stations]
+X = np.array((x, y))
+X = X.T
+
+
+try:
+    mongo_bulk = db.stations.initialize_ordered_bulk_op()
+    mongo_bulk.find({}).update({'$set': {'clusters': []}})
+
+    for n_clusters in reversed(range_clusters):
+
+        model = AgglomerativeClustering(linkage='ward', connectivity=None, n_clusters=n_clusters)
+        labels = model.fit_predict(X)
+
+        for label in range(len(np.unique(labels))):
+            cluster_assign = labels == label
+            cluster = X[cluster_assign]
+
+            average = np.average(cluster, 0)
+            middle = cluster[KDTree(cluster).query(average)[1]]
+
+            indexes = np.where((X == middle).all(axis=1))[0]
+            if len(indexes) > 1:
+                stations = list(db.stations.find({'_id': {
+                    '$in': [ids[index] for index in indexes.tolist()]
+                }}, {'last._id': 1}))
+                values = {station['_id']: station.get('last', {}).get('_id', 0) for station in stations}
+                station_id = max(values.keys(), key=(lambda k: values[k]))
+                if values[station_id] != 0:
+                    log.warning(f"Multiple 'middle' found, '{station_id}' has the latest measure")
+                else:
+                    log.warning(f"Ignoring '{ids[cluster_assign]}', stations have no measures")
+                    continue
+                index = np.where(ids == station_id)[0][0]
+            else:
+                index = indexes[0]
+            log.info(f'{n_clusters}: {ids[cluster_assign]} -> {ids[index]}')
+            mongo_bulk.find({'_id': ids[index]}).update({'$addToSet': {'clusters': int(n_clusters)}})
+
+    mongo_bulk.execute()
+except Exception as e:
+    log.exception(f'Error while creating clusters: {e}')

README.md

@@ -0,0 +1,13 @@
+version: 1
+disable_existing_loggers: True
+formatters:
+  console:
+    format: "%(levelname)s [%(name)s]: %(message)s"
+handlers:
+  console:
+    class: logging.StreamHandler
+    formatter: console
+    stream: ext://sys.stdout
+root:
+  handlers: [console]
+  level: INFO

clusters.py

@@ -0,0 +1,22 @@
+version: 1
+disable_existing_loggers: True
+formatters:
+  console:
+    format: "%(levelname)s [%(name)s]: %(message)s"
+  file:
+    format: "%(asctime)s %(levelname)s [%(name)s]: %(message)s"
+    datefmt: "%Y-%m-%dT%H:%M:%S%z"
+handlers:
+  console:
+    class: logging.StreamHandler
+    formatter: console
+    stream: ext://sys.stdout
+  file:
+    class: logging.handlers.TimedRotatingFileHandler
+    formatter: file
+    when: midnight
+    backupCount: 10
+
+root:
+  handlers: [console, file]
+  level: INFO

commons/__init__.py

@@ -0,0 +1,48 @@
+import math
+
+
+def dm_to_dd(s):
+    d, m = s.split('°')
+    dd = float(d) + float(m.strip()[:-1]) / 60
+    return dd
+
+
+def ch_to_wgs_lat(y, x):
+    """Convert CH y/x to WGS lat"""
+
+    # Converts military to civil and  to unit = 1000km
+    # Auxiliary values (% Bern)
+    y_aux = (y - 600000)/1000000
+    x_aux = (x - 200000)/1000000
+
+    # Process lat
+    lat = 16.9023892 \
+        + 3.238272 * x_aux \
+        - 0.270978 * math.pow(y_aux, 2) \
+        - 0.002528 * math.pow(x_aux, 2) \
+        - 0.0447 * math.pow(y_aux, 2) * x_aux \
+        - 0.0140 * math.pow(x_aux, 3)
+
+    # Unit 10000" to 1 " and converts seconds to degrees (dec)
+    lat = lat * 100/36
+    return lat
+
+
+def ch_to_wgs_lon(y, x):
+    """Convert CH y/x to WGS long"""
+
+    # Converts military to civil and  to unit = 1000km
+    # Auxiliary values (% Bern)
+    y_aux = (y - 600000)/1000000
+    x_aux = (x - 200000)/1000000
+
+    # Process long
+    lon = 2.6779094 \
+        + 4.728982 * y_aux \
+        + 0.791484 * y_aux * x_aux \
+        + 0.1306 * y_aux * math.pow(x_aux, 2) \
+        - 0.0436 * math.pow(y_aux, 3)
+
+    # Unit 10000" to 1 " and converts seconds to degrees (dec)
+    lon = lon * 100/36
+    return lon