Merge pull request #98 from winds-mobi/feature/ttl-index

Migrate from a capped collection to a ttl index

db-migration/migrate-measurement-collection-to-ttl.js

@@ -0,0 +1,16 @@
+db = connect(process.env.MONGODB_URL);
+
+const cursor = db.stations.find();
+while (cursor.hasNext()) {
+    const station = cursor.next();
+    const oldCollection = `${station._id}`;
+    const newCollection = `${station._id}-ttl`;
+
+    db[newCollection].createIndex({time: 1}, {expireAfterSeconds: 60 * 60 * 24 * 10});
+    db[oldCollection].aggregate([
+        {$set: {receivedAt: {$toDate: {$multiply: ["$time", 1000]}}}},
+        {$set: {time: {$toDate: {$multiply: ["$_id", 1000]}}}},
+        {$out: newCollection}]
+    );
+    db[newCollection].renameCollection(oldCollection, true);
+}

winds_mobi_provider/provider.py

@@ -77,7 +77,8 @@ def stations_collection(self):
 
     def measures_collection(self, station_id):
         if station_id not in self.collection_names:
-            self.mongo_db.create_collection(station_id, **{"capped": True, "size": 500000, "max": 5000})
+            self.mongo_db.create_collection(station_id)
+            self.mongo_db[station_id].create_index([("time", ASCENDING)], expireAfterSeconds=60 * 60 * 24 * 10)
             self.collection_names.append(station_id)
         return self.mongo_db[station_id]
 
@@ -403,7 +404,8 @@ def create_measure(
         if rain is not None:
             measure["rain"] = self.__to_rain(rain)
 
-        measure["time"] = arrow.now().int_timestamp
+        measure["time"] = arrow.get(measure["_id"]).datetime
+        measure["receivedAt"] = arrow.now().datetime
 
         fixes = self.mongo_db.stations_fix.find_one(for_station["_id"])
         if fixes and "measures" in fixes: