Skip to content

Implement file based cache #60

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 20 commits into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -2,4 +2,5 @@ node_modules
npm-debug.log
.DS_Store
test/out/*
test/cache/*
dist
10 changes: 10 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -46,6 +46,9 @@ height | Required | Height of the output image in px
paddingX | 0 | (optional) Minimum distance in px between map features and map border
paddingY | 0 | (optional) Minimum distance in px between map features and map border
tileUrl | | (optional) Tile server URL for the map base layer or `null` for empty base layer. `{x},{y},{z}` or `{quadkey}` supported.
tileCacheFolder | | (optional) When set to an existing folder, a file cache is used
tileCacheLifetime | 86400 | (optional) Time before tile in cache expire and will be reloaded
tileCacheAutoPurge | true | (optional) Should the Filebased TileCache automatically purged
tileSubdomains | [] | (optional) Subdomains of tile server, usage `['a', 'b', 'c']`
tileSize | 256 | (optional) Tile size in pixel
tileRequestTimeout | | (optional) Timeout for the tiles request
@@ -65,6 +68,7 @@ Method | Description
[addMultiPolygon](#addmultipolygon-options) | Adds a multipolygon to the map
[addCircle](#addcircle-options) | Adds a circle to the map
[addText](#addtext-options) | Adds text to the map
[clearCache](#clearcache) | Manually clear the base layer tile cache
[render](#render-center-zoom) | Renders the map and added features
[image.save](#imagesave-filename-outputoptions) | Saves the map image to a file
[image.buffer](#imagebuffer-mime-outputoptions) | Saves the map image to a buffer
@@ -267,6 +271,12 @@ center | | (optional) Set center of map to a specific coo
zoom | | (optional) Set a specific zoom level.

***
#### clearCache ()
clear the file based Tile cache.
Can be used, if tileCacheAutoPurge is set to false to clear the cache
```
map.clearCache();
```

#### image.save (fileName, [outputOptions])
Saves the image to a file in `fileName`.
40 changes: 40 additions & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
@@ -49,6 +49,7 @@
"@babel/preset-stage-2": "^7.8.3",
"@babel/register": "^7.13.16",
"chai": "^4.3.4",
"chai-image": "^3.0.0",
"eslint": "^7.27.0",
"eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-import": "^2.23.3",
122 changes: 117 additions & 5 deletions src/staticmaps.js
Original file line number Diff line number Diff line change
@@ -4,7 +4,9 @@ import find from 'lodash.find';
import uniqBy from 'lodash.uniqby';
import url from 'url';
import chunk from 'lodash.chunk';

import { createHash } from 'crypto';
import fs from 'fs';
import path from 'path';
import Image from './image';
import IconMarker from './marker';
import Polyline from './polyline';
@@ -29,6 +31,13 @@ class StaticMaps {
this.padding = [this.paddingX, this.paddingY];
this.tileUrl = 'tileUrl' in this.options ? this.options.tileUrl : 'https://tile.openstreetmap.org/{z}/{x}/{y}.png';
this.tileSize = this.options.tileSize || 256;

this.tileCacheFolder = this.options.tileCacheFolder || null;
this.tileCacheAutoPurge = typeof this.options.tileCacheAutoPurge !== 'undefined' ?
this.options.tileCacheAutoPurge : true;
this.tileCacheLifetime = this.options.tileCacheLifetime || 86400;
this.tileCacheHits = 0;

this.tileSubdomains = this.options.tileSubdomains || this.options.subdomains || [];
this.tileRequestTimeout = this.options.tileRequestTimeout;
this.tileRequestHeader = this.options.tileRequestHeader;
@@ -120,9 +129,48 @@ class StaticMaps {
this.drawBaselayer(),
this.loadMarker(),
]);

// when a cache folder is configured and auto purge is enable
// clear cache in 10% of all executions
if (this.tileCacheFolder !== null
&& this.tileCacheAutoPurge === true
&& Math.random() * 10 <= 1) {
this.clearCache();
}

return this.drawFeatures();
}

getTileCacheHits() {
return this.tileCacheHits;
}

async clearCache() {
if (this.tileCacheFolder !== null) {
const now = new Date().getTime();

fs.readdir(this.tileCacheFolder, (err, files) => {
files.forEach((file, index) => {
fs.stat(path.join(this.tileCacheFolder, file), (err, stat) => {
if (err) {
return console.error(err);
}

const fileMTime = new Date(stat.mtime).getTime() + this.tileCacheLifetime * 1000;

if (now > fileMTime) {
return fs.unlink(path.join(this.tileCacheFolder, file), (err) => {
if (err) {
return console.error(err);
}
});
}
});
});
});
}
}

/**
* calculate common extent of all current map features
*/
@@ -487,6 +535,7 @@ class StaticMaps {
* Fetching tile from endpoint
*/
async getTile(data) {

const options = {
url: data.url,
responseType: 'buffer',
@@ -496,21 +545,84 @@ class StaticMaps {
};

try {
const res = await got.get(options);
const { body, headers } = res;
let cacheFile = null;

if (this.tileCacheFolder !== null) {
const cacheKey = createHash('sha256').update(data.url).digest('hex');
cacheFile = path.join(this.tileCacheFolder, cacheKey);

if (fs.existsSync(cacheFile)) {
const stats = fs.statSync(cacheFile);

const seconds = (new Date().getTime() - stats.mtime) / 1000;

// If TTL expire, delete file
if (seconds < this.tileCacheLifetime) {
let cacheData;
try {
cacheData = JSON.parse(fs.readFileSync(cacheFile));
} catch(e) {
try {
cacheData = JSON.parse(fs.readFileSync(cacheFile));
} catch(e) {}
}

if(cacheData && cacheData.length > 0) {
try {
cacheData = Buffer.from(cacheData, 'base64');

if(cacheData && cacheData.length > 0) {
const responseContent = {
success: true,
tile: {
url: data.url,
box: data.box,
body: cacheData,
},
};

this.tileCacheHits++;

return responseContent;
}
} catch (e) {}
}
}

fs.rmSync(cacheFile);
}
}

let res = await got.get(options);
const { body, headers } = res;

const contentType = headers['content-type'];
if (!contentType.startsWith('image/')) throw new Error('Tiles server response with wrong data');
// console.log(headers);

return {
const responseContent = {
success: true,
tile: {
url: data.url,
box: data.box,
body,
},
};

if (this.tileCacheFolder !== null) {
fs.writeFile(cacheFile, JSON.stringify(responseContent.tile.body.toString('base64')), (err) => {
if (err) {
console.error(err);
}
// file written successfully
});

if (typeof responseContent.tile.body === 'string') {
responseContent.tile.body = Buffer.from(responseContent.tile.body, 'base64');
}
}

return responseContent;

} catch (error) {
return {
success: false,
Loading