Merge pull request #330 from linux-on-ibm-z/master-endian

Add support for big endian platforms

Author: kriszyp

index.js

@@ -1,6 +1,6 @@
 import { EventEmitter } from 'events';
 import { setExternals, setNativeFunctions, Dbi, version } from './native.js';
-import { arch, tmpdir, platform } from 'os';
+import { arch, tmpdir, platform, endianness } from 'os';
 import fs from 'fs';
 import { Encoder as MsgpackrEncoder } from 'msgpackr';
 import { WeakLRUCache } from 'weak-lru-cache';
@@ -14,7 +14,7 @@ setExternals({
 		if (process.getMaxListeners() < process.listenerCount('exit') + 8)
 			process.setMaxListeners(process.listenerCount('exit') + 8);
 		process.on('exit', callback);
-	},
+	}, isLittleEndian: endianness() == 'LE'
 });
 export { toBufferKey as keyValueToBuffer, compareKeys, compareKeys as compareKey, fromBufferKey as bufferToKeyValue } from 'ordered-binary';
 export { ABORT, IF_EXISTS, asBinary } from './write.js';
@@ -25,8 +25,25 @@ import { levelup } from './level.js';
 export { clearKeptObjects, version } from './native.js';
 import { nativeAddon } from './native.js';
 export let { noop } = nativeAddon;
-export const TIMESTAMP_PLACEHOLDER = new Uint8Array([1,1,1,1,0,0,0,0]);
-export const DIRECT_WRITE_PLACEHOLDER = new Uint8Array([1,1,1,2,0,0,0,0]);
+export const TIMESTAMP_PLACEHOLDER = (() => {
+	if (endianness() == 'BE') {
+		return new Uint8Array([0,0,0,0,1,1,1,1]);
+	} else {
+		return new Uint8Array([1,1,1,1,0,0,0,0]);
+	}
+})();
+export const DIRECT_WRITE_PLACEHOLDER = (() => {
+	if (endianness() == 'BE') {
+		return new Uint8Array([0,0,0,0,2,1,1,1]);
+	} else {
+		return new Uint8Array([1,1,1,2,0,0,0,0]);
+	}
+})();
+export function setSpecialWriteValue(destArray, placeholder, uint32Value) {
+	destArray.set(placeholder);
+	let uint32 = new Uint32Array(destArray.buffer, 0, 2);
+	endianness() == 'BE' ? uint32[0] = uint32Value : uint32[1] = uint32Value;
+}
 export { open, openAsClass, getLastVersion, allDbs, getLastTxnId } from './open.js';
 import { toBufferKey as keyValueToBuffer, compareKeys as compareKey, fromBufferKey as bufferToKeyValue } from 'ordered-binary';
 import { open, openAsClass, getLastVersion } from './open.js';

keys.js

@@ -1,12 +1,12 @@
-import { getAddress, orderedBinary } from './native.js';
+import { getAddress, orderedBinary, isLittleEndian } from './native.js';
 
 const REUSE_BUFFER_MODE = 512;
 const writeUint32Key = (key, target, start) => {
-	(target.dataView || (target.dataView = new DataView(target.buffer, 0, target.length))).setUint32(start, key, true);
+	(target.dataView || (target.dataView = new DataView(target.buffer, 0, target.length))).setUint32(start, key, isLittleEndian);
 	return start + 4;
 };
 const readUint32Key = (target, start) => {
-	return (target.dataView || (target.dataView = new DataView(target.buffer, 0, target.length))).getUint32(start, true);
+	return (target.dataView || (target.dataView = new DataView(target.buffer, 0, target.length))).getUint32(start, isLittleEndian);
 };
 const writeBufferKey = (key, target, start) => {
 	target.set(key, start);
@@ -70,7 +70,7 @@ function allocateSaveBuffer() {
 	saveDataAddress = saveBuffer.buffer.address;
 	// TODO: Conditionally only do this for key sequences?
 	saveDataView.setUint32(savePosition, 0xffffffff);
-	saveDataView.setFloat64(savePosition + 4, saveDataAddress, true); // save a pointer from the old buffer to the new address for the sake of the prefetch sequences
+	saveDataView.setFloat64(savePosition + 4, saveDataAddress, isLittleEndian); // save a pointer from the old buffer to the new address for the sake of the prefetch sequences
 	saveDataView = saveBuffer.dataView || (saveBuffer.dataView = new DataView(saveBuffer.buffer, saveBuffer.byteOffset, saveBuffer.byteLength));
 	savePosition = 0;
 }
@@ -103,7 +103,7 @@ export function saveKey(key, writeKey, saveTo, maxKeySize, flags) {
 		return saveKey(key, writeKey, saveTo, maxKeySize);
 	}
 	if (saveTo) {
-		saveDataView.setUint32(start, flags ? length | flags : length, true); // save the length
+		saveDataView.setUint32(start, flags ? length | flags : length, isLittleEndian); // save the length
 		saveTo.saveBuffer = saveBuffer;
 		savePosition = (savePosition + 12) & 0xfffffc;
 		return start + saveDataAddress;

native.js

@@ -16,6 +16,7 @@ export let Env,
 	fs,
 	os,
 	onExit,
+	isLittleEndian,
 	tmpdir,
 	lmdbError,
 	path,
@@ -131,4 +132,5 @@ export function setExternals(externals) {
 	tmpdir = externals.tmpdir;
 	os = externals.os;
 	onExit = externals.onExit;
+	isLittleEndian = externals.isLittleEndian;
 }

open.js

@@ -1,5 +1,5 @@
 import { Compression, getAddress, arch, fs, path as pathModule, lmdbError, EventEmitter, MsgpackrEncoder, Env,
-	Dbi, tmpdir, os, nativeAddon, version } from './native.js';
+	Dbi, tmpdir, os, nativeAddon, version, isLittleEndian } from './native.js';
 import { CachingStore, setGetLastVersion } from './caching.js';
 import { addReadMethods, makeReusableBuffer } from './read.js';
 import { addWriteMethods } from './write.js';
@@ -410,14 +410,14 @@ export function openAsClass(path, options) {
 }
 
 export function getLastVersion() {
-	return keyBytesView.getFloat64(16, true);
+	return keyBytesView.getFloat64(16, isLittleEndian);
 }
 export function setLastVersion(version) {
-	return keyBytesView.setFloat64(16, version, true);
+	return keyBytesView.setFloat64(16, version, isLittleEndian);
 }
 
 export function getLastTxnId() {
-	return keyBytesView.getUint32(32, true);
+	return keyBytesView.getUint32(32, isLittleEndian);
 }
 
 const KEY_BUFFER_SIZE = 4096;

read.js

@@ -23,6 +23,7 @@ import {
 	notifyUserCallbacks,
 	attemptLock,
 	unlock,
+	isLittleEndian
 } from './native.js';
 import { saveKey } from './keys.js';
 const IF_EXISTS = 3.542694326329068e-103;
@@ -108,11 +109,11 @@ export function addReadMethods(
 					);
 				if (rc == -30000)
 					// int32 overflow, read uint32
-					rc = this.lastSize = keyBytesView.getUint32(0, true);
+					rc = this.lastSize = keyBytesView.getUint32(0, isLittleEndian);
 				else if (rc == -30001) {
 					// shared buffer
-					this.lastSize = keyBytesView.getUint32(0, true);
-					let bufferId = keyBytesView.getUint32(4, true);
+					this.lastSize = keyBytesView.getUint32(0, isLittleEndian);
+					let bufferId = keyBytesView.getUint32(4, isLittleEndian);
 					let bytes = getMMapBuffer(bufferId, this.lastSize);
 					return asSafeBuffer ? Buffer.from(bytes) : bytes;
 				} else throw lmdbError(rc);
@@ -624,7 +625,7 @@ export function addReadMethods(
 								keyBytesView.setFloat64(
 									START_ADDRESS_POSITION,
 									startAddress,
-									true,
+									isLittleEndian,
 								);
 								endAddress = saveKey(
 									options.end,
@@ -645,7 +646,7 @@ export function addReadMethods(
 								keyBytesView.setFloat64(
 									START_ADDRESS_POSITION,
 									startAddress,
-									true,
+									isLittleEndian,
 								);
 								endAddress = saveKey(
 									options.end,
@@ -736,8 +737,8 @@ export function addReadMethods(
 						}
 						if (includeValues) {
 							let value;
-							lastSize = keyBytesView.getUint32(0, true);
-							let bufferId = keyBytesView.getUint32(4, true);
+							lastSize = keyBytesView.getUint32(0, isLittleEndian);
+							let bufferId = keyBytesView.getUint32(4, isLittleEndian);
 							let bytes;
 							if (bufferId) {
 								bytes = getMMapBuffer(bufferId, lastSize);
@@ -850,9 +851,9 @@ export function addReadMethods(
 				return; //undefined
 			}
 			return this.lastSize;
-			this.lastSize = keyBytesView.getUint32(0, true);
-			let bufferIndex = keyBytesView.getUint32(12, true);
-			lastOffset = keyBytesView.getUint32(8, true);
+			this.lastSize = keyBytesView.getUint32(0, isLittleEndian);
+			let bufferIndex = keyBytesView.getUint32(12, isLittleEndian);
+			lastOffset = keyBytesView.getUint32(8, isLittleEndian);
 			let buffer = buffers[bufferIndex];
 			let startOffset;
 			if (
@@ -1001,7 +1002,7 @@ export function addReadMethods(
 		if (!buffer) {
 			buffer = mmaps[bufferId] = getSharedBuffer(bufferId, env.address);
 		}
-		let offset = keyBytesView.getUint32(8, true);
+		let offset = keyBytesView.getUint32(8, isLittleEndian);
 		return new Uint8Array(buffer, offset, size);
 	}
 	function renewReadTxn(store) {
@@ -1152,7 +1153,7 @@ export function recordReadInstruction(
 	uint32Instructions[(start >> 2) + 3] = length; // save the length
 	uint32Instructions[(start >> 2) + 2] = dbi;
 	savePosition = (savePosition + 12) & 0xfffffc;
-	instructionsDataView.setFloat64(start, txnAddress, true);
+	instructionsDataView.setFloat64(start, txnAddress, isLittleEndian);
 	let callbackId = addReadCallback(() => {
 		let position = start >> 2;
 		let rc = thisInstructions[position];

src/env.cpp

@@ -12,6 +12,17 @@ using namespace Napi;
 #include <v8.h>
 #endif
 
+#if defined (__linux)
+#include <endian.h> // For __BYTE_ORDER, __BIG_ENDIAN
+#if __BYTE_ORDER == __BIG_ENDIAN
+#define BE64_TO_HOST(x) (x)
+#else
+#define BE64_TO_HOST(x) bswap_64((x))
+#endif
+#else
+#define BE64_TO_HOST(x) bswap_64((x))
+#endif
+
 MDB_txn* ExtendedEnv::prefetchTxns[20];
 pthread_mutex_t* ExtendedEnv::prefetchTxnsLock;
 env_tracking_t* EnvWrap::envTracking = EnvWrap::initTracking();
@@ -1080,10 +1091,10 @@ ExtendedEnv::~ExtendedEnv() {
 uint64_t ExtendedEnv::getNextTime() {
 	uint64_t next_time_int = next_time_double();
 	if (next_time_int == lastTime) next_time_int++;
-	return bswap_64(lastTime = next_time_int);
+	return BE64_TO_HOST(lastTime = next_time_int);
 }
 uint64_t ExtendedEnv::getLastTime() {
-	return bswap_64(lastTime);
+	return BE64_TO_HOST(lastTime);
 }
 
 NAPI_FUNCTION(getUserSharedBuffer) {

src/ordered-binary.cpp

@@ -34,8 +34,12 @@ int compareFast(const MDB_val *a, const MDB_val *b) {
             bVal = *((uint8_t*) dataB);
         } else {
             aVal = ntohl(*dataA);
+            #if __BYTE_ORDER == __BIG_ENDIAN
+            bVal = remaining == 2 ? *dataB & 0xffff0000 : *dataB & 0xffffff00;
+            #else
             bVal = remaining == 2 ? (*((uint8_t*) dataB) << 24) + (*((uint8_t*) dataB + 1) << 16) :
                 ntohl(*dataB & 0x00ffffff);
+            #endif
         }
         if (aVal > bVal)
             return 1;

test/index.test.js

@@ -30,6 +30,7 @@ import {
 } from '../node-index.js';
 import { openAsClass } from '../open.js';
 import { RangeIterable } from '../util/RangeIterable.js';
+import { setSpecialWriteValue } from '../index.js';
 const require = createRequire(import.meta.url);
 // we don't always test CJS because it messes up debugging in webstorm (and I am not about to give the awesomeness
 // that is webstorm debugging)
@@ -1510,8 +1511,7 @@ describe('lmdb-js', function () {
 					}),
 				);
 				let value = Buffer.alloc(16, 3);
-				value.set(TIMESTAMP_PLACEHOLDER);
-				value[4] = 0;
+				setSpecialWriteValue(value, TIMESTAMP_PLACEHOLDER, 0);
 				await dbBinary.put(1, value, {
 					instructedWrite: true,
 				});
@@ -1523,8 +1523,7 @@ describe('lmdb-js', function () {
 				should.equal(returnedValue[9], 3);
 
 				value = Buffer.alloc(16, 3);
-				value.set(TIMESTAMP_PLACEHOLDER);
-				value[4] = 1; // assign previous
+				setSpecialWriteValue(value, TIMESTAMP_PLACEHOLDER, 1); // assign previous
 
 				await dbBinary.put(1, value, {
 					instructedWrite: true,
@@ -1653,8 +1652,7 @@ describe('lmdb-js', function () {
 				let returnedValue = dbBinary.get(1);
 				should.equal(returnedValue[2], 4);
 				value = Buffer.alloc(12, 3);
-				value.set(DIRECT_WRITE_PLACEHOLDER);
-				value[4] = 2;
+				setSpecialWriteValue(value, DIRECT_WRITE_PLACEHOLDER, 2);
 				value.set([1, 2, 3, 4], 8);
 
 				await dbBinary.put(1, value, {
@@ -1667,8 +1665,7 @@ describe('lmdb-js', function () {
 
 				// this should always trigger the full put operation
 				value = Buffer.alloc(18, 3);
-				value.set(DIRECT_WRITE_PLACEHOLDER);
-				value[4] = 2;
+				setSpecialWriteValue(value, DIRECT_WRITE_PLACEHOLDER, 2);
 				value.set([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 8);
 
 				await dbBinary.put(1, value, {
@@ -1697,8 +1694,7 @@ describe('lmdb-js', function () {
 					let returnedValue = dbBinary.get(1);
 					let updated_byte = i % 200;
 					value = Buffer.alloc(32, updated_byte);
-					value.set(DIRECT_WRITE_PLACEHOLDER);
-					value[4] = 2;
+					setSpecialWriteValue(value, DIRECT_WRITE_PLACEHOLDER, 2);
 					let promise = dbBinary.put(1, value, {
 						instructedWrite: true,
 					});
@@ -1754,8 +1750,7 @@ describe('lmdb-js', function () {
 					let returnedValue = dbBinary.get(1);
 					let updated_byte = i % 200;
 					value = Buffer.alloc(16, updated_byte);
-					value.set(DIRECT_WRITE_PLACEHOLDER);
-					value[4] = 2;
+					setSpecialWriteValue(value, DIRECT_WRITE_PLACEHOLDER, 2);
 					let promise = dbBinary.put(1, value, {
 						instructedWrite: true,
 					});

test/threads.cjs

@@ -58,7 +58,7 @@ if (isMainThread) {
 
 			setTimeout(() => {
 				worker.terminate();
-			}, 100);
+			}, 500);
 			if (messages.length === workerCount) {
 				db.close();
 				for (var i = 0; i < messages.length; i++) {