This repository has been archived by the owner on Jun 13, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 19
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(batch processing): traces queue + batch sending + byte size limit (
#373) * feat(batch processing): trace queue + events emiter + batch * feat(batch processing): batch config + example + fixed sendTrace * feat(batch processing): batch byte size limit * feat(batch processing): init queue * feat(batch processing): log fixes + release on process exit * feat(batch processing): queue bytes size limit + config
- Loading branch information
Itay Katz
authored
Nov 16, 2020
1 parent
c961ac9
commit ec434b6
Showing
10 changed files
with
550 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,4 +5,7 @@ issues/ | |
| src/resource_utils/sql_utils.js | ||
| **/.DS_Store | ||
| .vscode | ||
| .env | ||
| .env | ||
| .pytest_cache | ||
| *.cpuprofile | ||
| tenna_releases | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| const epsagon = require('../src/index'); | ||
| const http = require('http'); | ||
| const { Console } = require('console'); | ||
|
|
||
|
|
||
| epsagon.init({ | ||
| token: process.env.EPSAGON_TOKEN, | ||
| appName: 'batch-test', | ||
| metadataOnly: false, | ||
| sendBatch: true, | ||
| batchSize: 5000, | ||
| maxBatchSizeBytes: 5000000, | ||
| maxTraceWait: 5000 // not in use | ||
| }); | ||
|
|
||
| epsagon.init({ | ||
| token: process.env.EPSAGON_TOKEN, | ||
| appName: 'batch-test', | ||
| metadataOnly: false, | ||
| sendBatch: true, | ||
| batchSize: 5000, | ||
| }); | ||
|
|
||
| function doRequest(options) { | ||
| return new Promise ((resolve, reject) => { | ||
| let req = http.request(options); | ||
|
|
||
| req.on('response', res => { | ||
| resolve(res); | ||
| }); | ||
|
|
||
| req.on('error', err => { | ||
| resolve(err); | ||
| }); | ||
| }); | ||
| } | ||
|
|
||
|
|
||
| async function testAsyncFunction() { | ||
| const options = { | ||
| host: 'localhost', | ||
| method: 'GET', | ||
| }; | ||
| doRequest(options) | ||
| console.log("logging something") | ||
| } | ||
|
|
||
|
|
||
| const wrappedAsyncTestFunction = epsagon.nodeWrapper(testAsyncFunction); | ||
|
|
||
| async function main (){ | ||
| await wrappedAsyncTestFunction() | ||
| await wrappedAsyncTestFunction() | ||
|
|
||
| await wrappedAsyncTestFunction() | ||
|
|
||
| await wrappedAsyncTestFunction() | ||
|
|
||
| await wrappedAsyncTestFunction() | ||
| await wrappedAsyncTestFunction() | ||
| await wrappedAsyncTestFunction() | ||
| await wrappedAsyncTestFunction() | ||
| await wrappedAsyncTestFunction() | ||
| await wrappedAsyncTestFunction() | ||
| await wrappedAsyncTestFunction() | ||
| await wrappedAsyncTestFunction() | ||
|
|
||
| await Promise.all([ | ||
| wrappedAsyncTestFunction(), | ||
| wrappedAsyncTestFunction()] | ||
|
|
||
| ) | ||
| } | ||
|
|
||
|
|
||
|
|
||
|
|
||
| main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,207 @@ | ||
| /** | ||
| * @fileoverview The traces queue, cunsume traces and sends in batches | ||
| */ | ||
| const EventEmitter = require('events'); | ||
| const axios = require('axios'); | ||
| const https = require('https'); | ||
| const http = require('http'); | ||
| const utils = require('../src/utils.js'); | ||
| const config = require('./config.js'); | ||
|
|
||
|
|
||
| /** | ||
| * Session for the post requests to the collector | ||
| */ | ||
| const session = axios.create({ | ||
| headers: { Authorization: `Bearer ${config.getConfig().token}` }, | ||
| timeout: config.getConfig().sendTimeout, | ||
| httpAgent: new http.Agent({ keepAlive: true }), | ||
| httpsAgent: new https.Agent({ keepAlive: true }), | ||
| }); | ||
|
|
||
| /** | ||
| * Post given batch to epsagon's infrastructure. | ||
| * @param {*} batchObject The batch data to send. | ||
| * @returns {Promise} a promise that is resolved after the batch is posted. | ||
| */ | ||
| async function postBatch(batchObject) { | ||
| utils.debugLog(`[QUEUE] Posting batch to ${config.getConfig().traceCollectorURL}...`); | ||
| const cancelTokenSource = axios.CancelToken.source(); | ||
| const handle = setTimeout(() => { | ||
| cancelTokenSource.cancel('Timeout sending batch!'); | ||
| }, config.getConfig().sendTimeout); | ||
|
|
||
| return session.post( | ||
| config.getConfig().traceCollectorURL, | ||
| batchObject, | ||
| { | ||
| cancelToken: cancelTokenSource.token, | ||
| } | ||
| ).then((res) => { | ||
| clearTimeout(handle); | ||
| utils.debugLog('[QUEUE] Batch posted!'); | ||
| return res; | ||
| }).catch((err) => { | ||
| clearTimeout(handle); | ||
| if (err.config && err.config.data) { | ||
| utils.debugLog(`[QUEUE] Error sending trace. Batch size: ${err.config.data.length}`); | ||
| } else { | ||
| utils.debugLog(`[QUEUE] Error sending trace. Error: ${err}`); | ||
| } | ||
| utils.debugLog(`[QUEUE] ${err ? err.stack : err}`); | ||
| return err; | ||
| }); | ||
| } | ||
|
|
||
| /** | ||
| * The trace queue class | ||
| * @param {function} batchSender function to send batch traces | ||
| */ | ||
| class TraceQueue extends EventEmitter.EventEmitter { | ||
| /** | ||
| * EventEmitter class | ||
| */ | ||
| constructor() { | ||
| super(); | ||
| this.batchSender = postBatch; | ||
| this.initQueue(); | ||
| } | ||
|
|
||
| /** | ||
| * Update the queue config | ||
| */ | ||
| updateConfig() { | ||
| this.maxBatchSizeBytes = config.getConfig().maxBatchSizeBytes; | ||
| this.batchSize = config.getConfig().batchSize; | ||
| this.maxQueueSizeBytes = config.getConfig().maxQueueSizeBytes; | ||
| } | ||
|
|
||
| /** | ||
| * Init queue event listners | ||
| */ | ||
| initQueue() { | ||
| this.updateConfig(); | ||
| this.removeAllListeners(); | ||
| this.flush(); | ||
| this.on('traceQueued', function traceQueued() { | ||
| if (this.byteSizeLimitReached()) { | ||
| utils.debugLog(`[QUEUE] Queue Byte size reached ${this.currentByteSize} Bytes, releasing batch...`); | ||
| this.emit('releaseRequest', Math.max(this.currentSize - 1, 1)); | ||
| } else if (this.batchSizeReached()) { | ||
| utils.debugLog(`[QUEUE] Queue size reached ${this.currentSize}, releasing batch... `); | ||
| this.emit('releaseRequest'); | ||
| } | ||
| return this; | ||
| }); | ||
|
|
||
| this.on('releaseRequest', function releaseRequest(count = this.batchSize) { | ||
| try { | ||
| const batch = this.queue.splice(0, count); | ||
| utils.debugLog('[QUEUE] Releasing batch...'); | ||
| this.subtractFromCurrentByteSize(batch); | ||
| this.emit('batchReleased', batch); | ||
| } catch (err) { | ||
| utils.debugLog('[QUEUE] Failed releasing batch!'); | ||
| utils.debugLog(`[QUEUE] ${err}`); | ||
| } | ||
| return this; | ||
| }); | ||
|
|
||
| this.on('batchReleased', async function batchReleased(batch) { | ||
| utils.debugLog('[QUEUE] Sending batch...'); | ||
| const batchJSON = batch.map(trace => trace.traceJSON); | ||
| this.batchSender(batchJSON); | ||
| }); | ||
| process.on('exit', function releaseAndClearQueue() { | ||
| this.emit('releaseRequest'); | ||
| this.removeAllListeners(); | ||
| }); | ||
| } | ||
|
|
||
| /** | ||
| * Push trace to queue, emit event, and check if queue max queue length reached, | ||
| * if it does, send batch. | ||
| * @param {object} traceJson Trace JSON | ||
| * @returns {TraceQueue} This trace queue | ||
| */ | ||
| push(traceJson) { | ||
| try { | ||
| if (this.currentByteSize >= this.maxQueueSizeBytes) { | ||
| utils.debugLog(`[QUEUE] Discardig trace, queue size reached max size of ${this.currentByteSize} Bytes`); | ||
| return this; | ||
| } | ||
| const timestamp = Date.now(); | ||
| const json = traceJson; | ||
| const string = JSON.stringify(json); | ||
| const byteLength = string.length; | ||
| // eslint-disable-next-line object-curly-newline | ||
| const trace = { json, string, byteLength, timestamp }; | ||
| this.queue.push(trace); | ||
| this.addToCurrentByteSize([trace]); | ||
| utils.debugLog(`[QUEUE] Trace size ${byteLength} Bytes pushed to queue`); | ||
| utils.debugLog(`[QUEUE] Queue size: ${this.currentSize} traces, total size of ${this.currentByteSize} Bytes`); | ||
| this.emit('traceQueued', trace); | ||
| } catch (err) { | ||
| utils.debugLog(`[QUEUE] Failed pushing trace to queue: ${err}`); | ||
| } | ||
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * add given trace byte size to total byte size | ||
| * @param {Array} traces Trace object array | ||
| */ | ||
| addToCurrentByteSize(traces) { | ||
| traces.forEach((trace) => { | ||
| this.currentByteSize += trace.byteLength; | ||
| }); | ||
| } | ||
|
|
||
| /** | ||
| * subtract given trace byte size to total byte size | ||
| * @param {Array} traces Trace object array | ||
| */ | ||
| subtractFromCurrentByteSize(traces) { | ||
| traces.forEach((trace) => { | ||
| this.currentByteSize -= trace.byteLength; | ||
| this.currentByteSize = Math.max(this.currentByteSize, 0); | ||
| }); | ||
| } | ||
|
|
||
| /** | ||
| * Queue size getter | ||
| * @returns {Number} Queue length | ||
| */ | ||
| get currentSize() { | ||
| return this.queue.length; | ||
| } | ||
|
|
||
|
|
||
| /** | ||
| * Checks if queue size reached batch size | ||
| * @returns {Boolean} Indicator for if current queue size is larger than batch size definition | ||
| */ | ||
| batchSizeReached() { | ||
| return this.currentSize >= this.batchSize; | ||
| } | ||
|
|
||
| /** | ||
| * Checks if queue byte size reached its limit | ||
| * @returns {Boolean} Indicator for if current queue byte size is larger than byte size definition | ||
| */ | ||
| byteSizeLimitReached() { | ||
| return this.currentByteSize >= this.maxBatchSizeBytes; | ||
| } | ||
|
|
||
| /** | ||
| * Flush queue | ||
| */ | ||
| flush() { | ||
| this.queue = []; | ||
| this.currentByteSize = 0; | ||
| } | ||
| } | ||
|
|
||
| const traceQueue = new TraceQueue(); | ||
|
|
||
| module.exports.getInstance = () => traceQueue; |
Oops, something went wrong.