Skip to content

Commit 8a5d225

Browse files
committed
Migration to github winds-mobi organization
0 parents  commit 8a5d225

38 files changed

+6564
-0
lines changed

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
*.pyc
2+
/.idea/
3+
/local_settings.py

LICENSE.txt

Lines changed: 661 additions & 0 deletions
Large diffs are not rendered by default.

Pipfile

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
[[source]]
2+
url = "https://pypi.org/simple"
3+
verify_ssl = true
4+
name = "pypi"
5+
6+
[packages]
7+
arrow = "==0.12.1"
8+
cachetools = "==1.1.3"
9+
lxml = "==4.0.0"
10+
metar = "==1.6.0"
11+
mysqlclient = "==1.3.6"
12+
numpy = "==1.12.1"
13+
pint = "==0.8.1"
14+
pyaml = "==16.9.0"
15+
pymongo = "==3.0.3"
16+
pytz = "==2018.5"
17+
redis = "==2.10.3"
18+
requests = "==2.19.1"
19+
sentry-sdk = "==0.6.5"
20+
scikit-learn = "==0.20.0"
21+
scipy = "==0.19.0"
22+
tenacity = "==5.0.2"
23+
xmltodict = "==0.10.2"
24+
25+
[dev-packages]
26+
"flake8" = "*"
27+
"flake8-quotes" = "*"
28+
29+
[requires]
30+
python_version = "3.6"

Pipfile.lock

Lines changed: 353 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

README.md

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
winds.mobi - real-time weather observations
2+
===========================================
3+
4+
[winds.mobi](http://winds.mobi): Paraglider pilot, kitesurfer, check real-time weather conditions of your favorite spots
5+
on your smartphone, your tablet or your computer.
6+
7+
Follow this project on:
8+
- [Facebook](https://www.facebook.com/WindsMobi/)
9+
10+
winds-mobi-providers
11+
--------------------
12+
13+
Python 3.6 cronjobs that get the weather data from different providers and save it into mongodb. This project use
14+
Google Cloud APIs to compute any missing station details (altitude, name, timezone, ...). Google Cloud API results are
15+
cached with redis.
16+
17+
### How to build
18+
19+
- pipenv install
20+
21+
Licensing
22+
---------
23+
24+
Please see the file called [LICENSE.txt](https://github.com/winds-mobi/winds-mobi-providers/blob/master/LICENSE.txt)

clusters.py

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import argparse
2+
from datetime import datetime
3+
4+
import numpy as np
5+
from commons.provider import get_logger
6+
from pymongo import MongoClient, uri_parser
7+
from scipy.spatial import KDTree
8+
from settings import MONGODB_URL
9+
from sklearn.cluster import AgglomerativeClustering
10+
11+
log = get_logger('clusters')
12+
13+
parser = argparse.ArgumentParser(description='Save clusters membership in mongodb')
14+
parser.add_argument('--num', type=int, default=50, help='Specify the number of cluster levels [default: %(default)s]')
15+
args = vars(parser.parse_args())
16+
17+
uri = uri_parser.parse_uri(MONGODB_URL)
18+
client = MongoClient(uri['nodelist'][0][0], uri['nodelist'][0][1])
19+
db = client.windmobile
20+
21+
now = datetime.now().timestamp()
22+
all_stations = list(db.stations.find({
23+
'status': {'$ne': 'hidden'},
24+
'last._id': {'$gt': now - 30 * 24 * 3600}
25+
}))
26+
range_clusters = np.geomspace(20, len(all_stations), num=args['num'], dtype=np.int)
27+
28+
ids = np.array([station['_id'] for station in all_stations])
29+
30+
x = [station['loc']['coordinates'][0] for station in all_stations]
31+
y = [station['loc']['coordinates'][1] for station in all_stations]
32+
X = np.array((x, y))
33+
X = X.T
34+
35+
36+
try:
37+
mongo_bulk = db.stations.initialize_ordered_bulk_op()
38+
mongo_bulk.find({}).update({'$set': {'clusters': []}})
39+
40+
for n_clusters in reversed(range_clusters):
41+
42+
model = AgglomerativeClustering(linkage='ward', connectivity=None, n_clusters=n_clusters)
43+
labels = model.fit_predict(X)
44+
45+
for label in range(len(np.unique(labels))):
46+
cluster_assign = labels == label
47+
cluster = X[cluster_assign]
48+
49+
average = np.average(cluster, 0)
50+
middle = cluster[KDTree(cluster).query(average)[1]]
51+
52+
indexes = np.where((X == middle).all(axis=1))[0]
53+
if len(indexes) > 1:
54+
stations = list(db.stations.find({'_id': {
55+
'$in': [ids[index] for index in indexes.tolist()]
56+
}}, {'last._id': 1}))
57+
values = {station['_id']: station.get('last', {}).get('_id', 0) for station in stations}
58+
station_id = max(values.keys(), key=(lambda k: values[k]))
59+
if values[station_id] != 0:
60+
log.warning(f"Multiple 'middle' found, '{station_id}' has the latest measure")
61+
else:
62+
log.warning(f"Ignoring '{ids[cluster_assign]}', stations have no measures")
63+
continue
64+
index = np.where(ids == station_id)[0][0]
65+
else:
66+
index = indexes[0]
67+
log.info(f'{n_clusters}: {ids[cluster_assign]} -> {ids[index]}')
68+
mongo_bulk.find({'_id': ids[index]}).update({'$addToSet': {'clusters': int(n_clusters)}})
69+
70+
mongo_bulk.execute()
71+
except Exception as e:
72+
log.exception(f'Error while creating clusters: {e}')

commons/__init__.py

Whitespace-only changes.

commons/logging_console.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
version: 1
2+
disable_existing_loggers: True
3+
formatters:
4+
console:
5+
format: "%(levelname)s [%(name)s]: %(message)s"
6+
handlers:
7+
console:
8+
class: logging.StreamHandler
9+
formatter: console
10+
stream: ext://sys.stdout
11+
root:
12+
handlers: [console]
13+
level: INFO

commons/logging_file.yml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
version: 1
2+
disable_existing_loggers: True
3+
formatters:
4+
console:
5+
format: "%(levelname)s [%(name)s]: %(message)s"
6+
file:
7+
format: "%(asctime)s %(levelname)s [%(name)s]: %(message)s"
8+
datefmt: "%Y-%m-%dT%H:%M:%S%z"
9+
handlers:
10+
console:
11+
class: logging.StreamHandler
12+
formatter: console
13+
stream: ext://sys.stdout
14+
file:
15+
class: logging.handlers.TimedRotatingFileHandler
16+
formatter: file
17+
when: midnight
18+
backupCount: 10
19+
20+
root:
21+
handlers: [console, file]
22+
level: INFO

commons/projections.py

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import math
2+
3+
4+
def dm_to_dd(s):
5+
d, m = s.split('°')
6+
dd = float(d) + float(m.strip()[:-1]) / 60
7+
return dd
8+
9+
10+
def ch_to_wgs_lat(y, x):
11+
"""Convert CH y/x to WGS lat"""
12+
13+
# Converts military to civil and to unit = 1000km
14+
# Auxiliary values (% Bern)
15+
y_aux = (y - 600000)/1000000
16+
x_aux = (x - 200000)/1000000
17+
18+
# Process lat
19+
lat = 16.9023892 \
20+
+ 3.238272 * x_aux \
21+
- 0.270978 * math.pow(y_aux, 2) \
22+
- 0.002528 * math.pow(x_aux, 2) \
23+
- 0.0447 * math.pow(y_aux, 2) * x_aux \
24+
- 0.0140 * math.pow(x_aux, 3)
25+
26+
# Unit 10000" to 1 " and converts seconds to degrees (dec)
27+
lat = lat * 100/36
28+
return lat
29+
30+
31+
def ch_to_wgs_lon(y, x):
32+
"""Convert CH y/x to WGS long"""
33+
34+
# Converts military to civil and to unit = 1000km
35+
# Auxiliary values (% Bern)
36+
y_aux = (y - 600000)/1000000
37+
x_aux = (x - 200000)/1000000
38+
39+
# Process long
40+
lon = 2.6779094 \
41+
+ 4.728982 * y_aux \
42+
+ 0.791484 * y_aux * x_aux \
43+
+ 0.1306 * y_aux * math.pow(x_aux, 2) \
44+
- 0.0436 * math.pow(y_aux, 3)
45+
46+
# Unit 10000" to 1 " and converts seconds to degrees (dec)
47+
lon = lon * 100/36
48+
return lon

0 commit comments

Comments
 (0)