Commit 20d7e548 authored by Aral Balkan's avatar Aral Balkan
Browse files

Merge branch 'esm'

parents 176d51df 24f79ffe
const JSDB = require('../..')
import JSDB from '../../index.js'
const db = JSDB.open('db')
......
......@@ -3,7 +3,7 @@
// JavaScript Database (JSDB)
// ══════════════════════════
//
// Copyright ⓒ 2020 Aral Balkan. Licensed under AGPLv3 or later.
// Copyright ⓒ 2020-2021 Aral Balkan. Licensed under AGPLv3 or later.
// Shared with ♥ by the Small Technology Foundation.
//
// To use:
......@@ -33,4 +33,4 @@
//
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
module.exports = require('./lib/JSDB')
export { default } from './lib/JSDB.js'
......@@ -2,7 +2,7 @@
//
// DataProxy class.
//
// Copyright ⓒ 2020 Aral Balkan. Licensed under AGPLv3 or later.
// Copyright ⓒ 2020-2021 Aral Balkan. Licensed under AGPLv3 or later.
// Shared with ♥ by the Small Technology Foundation.
//
// Like this? Fund us!
......@@ -10,15 +10,14 @@
//
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
const JSDF = require('./JSDF')
const { needsToBeProxified, quoteKeyIfNotNumeric } = require('./Util')
// Note: further circular module references at end of file:
// - IncompleteQueryProxy
// - QueryProxy
import JSDF from './JSDF.js'
import { needsToBeProxified, quoteKeyIfNotNumeric } from './Util.js'
import IncompleteQueryProxy from './IncompleteQueryProxy.js'
import QueryProxy from './QueryProxy.js'
const variableReference = (id, property) => `${id}[${quoteKeyIfNotNumeric(property)}]`
class DataProxy {
export default class DataProxy {
//
// Class.
......@@ -144,8 +143,3 @@ class DataProxy {
return true
}
}
module.exports = DataProxy
const IncompleteQueryProxy = require('./IncompleteQueryProxy')
const QueryProxy = require('./QueryProxy')
......@@ -21,14 +21,15 @@
// Like this? Fund us!
// https://small-tech.org/fund-us
//
// Copyright ⓒ 2020 Aral Balkan. Licensed under AGPLv3 or later.
// Copyright ⓒ 2020-2021 Aral Balkan. Licensed under AGPLv3 or later.
// Shared with ♥ by the Small Technology Foundation.
//
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
const QueryOperators = require('./QueryOperators')
import QueryOperators from './QueryOperators.js'
import QueryProxy from './QueryProxy.js'
class IncompleteQueryProxy {
export default class IncompleteQueryProxy {
constructor (table, data, property) {
this.table = table
......@@ -73,7 +74,3 @@ class IncompleteQueryProxy {
}
}
}
module.exports = IncompleteQueryProxy
const QueryProxy = require('./QueryProxy')
......@@ -2,7 +2,7 @@
//
// JSDB class.
//
// Copyright ⓒ 2020 Aral Balkan. Licensed under AGPLv3 or later.
// Copyright ⓒ 2020-2021 Aral Balkan. Licensed under AGPLv3 or later.
// Shared with ♥ by the Small Technology Foundation.
//
// To use:
......@@ -14,21 +14,21 @@
//
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
const fs = require('fs-extra')
const path = require('path')
const { log } = require('./Util')
const asyncForEach = require('./async-foreach')
import fs from 'fs'
import path from 'path'
import { log } from './Util.js'
import asyncForEach from './async-foreach.js'
const JSTable = require('./JSTable')
import JSTable from './JSTable.js'
class JSDB {
export default class JSDB {
//
// Class.
//
static #isBeingInstantiatedByFactoryMethod = false
static #openDatabases = {}
static isBeingInstantiatedByFactoryMethod = false
static openDatabases = {}
//
// Public.
......@@ -38,12 +38,12 @@ class JSDB {
// returns the reference.
static open (basePath, options) {
basePath = path.resolve(basePath)
if (this.#openDatabases[basePath] == undefined) {
this.#isBeingInstantiatedByFactoryMethod = true
this.#openDatabases[basePath] = new this(basePath, options)
this.#isBeingInstantiatedByFactoryMethod = false
if (this.openDatabases[basePath] == undefined) {
this.isBeingInstantiatedByFactoryMethod = true
this.openDatabases[basePath] = new this(basePath, options)
this.isBeingInstantiatedByFactoryMethod = false
}
return this.#openDatabases[basePath]
return this.openDatabases[basePath]
}
//
......@@ -54,9 +54,8 @@ class JSDB {
// Instance.
//
#tableDataProxies = []
#tableNames = []
tableDataProxies = []
tableNames = []
constructor (basePath, options = {
deleteIfExists: false
......@@ -64,7 +63,7 @@ class JSDB {
// This class can only be instantiated via the open() factory method.
// This is to ensure that multiple instances of the same database cannot be opened at the
// same time, thereby leading to data corruption.
if (!JSDB.#isBeingInstantiatedByFactoryMethod) {
if (!JSDB.isBeingInstantiatedByFactoryMethod) {
throw new Error('The JSDB class cannot be directly instantiated. Please use the JSDB.open() factory method instead.')
}
......@@ -75,7 +74,7 @@ class JSDB {
if (options.deleteIfExists) {
log(` 💾 ❨JSDB❩ Fresh database requested at ${basePath}; existing database is being deleted.`)
fs.removeSync(basePath)
fs.rmSync(basePath, {recursive: true, force: true})
}
if (fs.existsSync(basePath)) {
......@@ -83,7 +82,7 @@ class JSDB {
this.loadTables()
} else {
log(` 💾 ❨JSDB❩ No database found at ${basePath}; creating it.`)
fs.mkdirpSync(basePath)
fs.mkdirSync(basePath, {recursive: true})
}
// NB. we are returning the data proxy, not an
......@@ -95,8 +94,8 @@ class JSDB {
onTableDelete (tableDataProxy) {
const nameOfTableToRemove = tableDataProxy.__table__.tableName
log(` 💾 ❨JSDB❩ Removing table ${nameOfTableToRemove} from database…`)
this.#tableDataProxies = this.#tableDataProxies.filter(dataProxy => dataProxy !== tableDataProxy)
this.#tableNames = this.#tableNames.filter(tableName => tableName !== nameOfTableToRemove)
this.tableDataProxies = this.tableDataProxies.filter(dataProxy => dataProxy !== tableDataProxy)
this.tableNames = this.tableNames.filter(tableName => tableName !== nameOfTableToRemove)
tableDataProxy.__table__.removeListener('delete', this.onTableDelete.bind(this))
this.dataProxy[nameOfTableToRemove] = null
log(` 💾 ❨JSDB❩ ╰─ Table in ${nameOfTableToRemove} removed from database.`)
......@@ -113,8 +112,8 @@ class JSDB {
const tableDataProxy = new JSTable(tablePath)
tableDataProxy.__table__.addListener('delete', this.onTableDelete.bind(this))
this.dataProxy[tableName] = tableDataProxy
this.#tableNames.push(tableName)
this.#tableDataProxies.push(tableDataProxy)
this.tableNames.push(tableName)
this.tableDataProxies.push(tableDataProxy)
})
this.loadingTables = false
}
......@@ -135,17 +134,16 @@ class JSDB {
if (property === 'close') {
return async function () {
log(` 💾 ❨JSDB❩ Closing database at ${this.basePath}…`)
await asyncForEach(this.#tableDataProxies, async tableDataProxy => {
await asyncForEach(this.tableDataProxies, async tableDataProxy => {
await tableDataProxy.__table__.close()
})
JSDB.#openDatabases[this.basePath] = null
JSDB.openDatabases[this.basePath] = null
log(` 💾 ❨JSDB❩ ╰─ Closed database at ${this.basePath}.`)
}.bind(this)
}
return Reflect.get(...arguments)
}
setHandler (target, property, value, receiver) {
//
// Only objects (including custom objects) and arrays are allowed at
......@@ -171,7 +169,7 @@ class JSDB {
// If we’re initially loading tables, do not attempt to create a new table.
if (!this.loadingTables) {
if (this.#tableNames.includes(property)) {
if (this.tableNames.includes(property)) {
// Table already exists. You cannot replace it by setting a new value
// as this is a synchronous operation that involves closing the current
// writeStream, deleting the table, and creating a new table. To replace a table,
......@@ -184,13 +182,11 @@ class JSDB {
const tablePath = path.join(this.basePath, tableFileName)
value = new JSTable(tablePath, value)
value.__table__.addListener('delete', this.onTableDelete.bind(this))
this.#tableDataProxies.push(value)
this.#tableNames.push(property)
this.tableDataProxies.push(value)
this.tableNames.push(property)
}
Reflect.set(target, property, value, receiver)
return true
}
}
module.exports = JSDB
......@@ -2,7 +2,7 @@
//
// JSDF class.
//
// Copyright ⓒ 2020 Aral Balkan. Licensed under AGPLv3 or later.
// Copyright ⓒ 2020-2021 Aral Balkan. Licensed under AGPLv3 or later.
// Shared with ♥ by the Small Technology Foundation.
//
// Recursively serialises a JavaScript data structure into JavaScript Data Format (a series
......@@ -13,10 +13,9 @@
//
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
const { quoteKeyIfNotNumeric } = require('./Util')
class JSDF {
import { quoteKeyIfNotNumeric } from './Util.js'
export default class JSDF {
static serialise (value, key, parentType = null) {
// Check that key is valid.
if (key === undefined || key === null) {
......@@ -176,5 +175,3 @@ class JSDF {
return serialisedStatement
}
}
module.exports = JSDF
......@@ -2,7 +2,7 @@
//
// JSTable class.
//
// Copyright ⓒ 2020 Aral Balkan. Licensed under AGPLv3 or later.
// Copyright ⓒ 2020-2021 Aral Balkan. Licensed under AGPLv3 or later.
// Shared with ♥ by the Small Technology Foundation.
//
// Each JSTable is kept in its own JavaScript Data Format (JSDF) file – an append-only
......@@ -13,21 +13,21 @@
//
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
const fs = require('fs-extra')
const path = require('path')
const EventEmitter = require('events')
import fs from 'fs'
import fsPromises from 'fs/promises'
import path from 'path'
import EventEmitter from 'events'
const DataProxy = require('./DataProxy')
const JSDF = require('./JSDF')
import DataProxy from './DataProxy.js'
import JSDF from './JSDF.js'
const { log } = require('./Util')
const Time = require('./Time')
const { performance } = require('perf_hooks')
import { log } from './Util.js'
import Time from './Time.js'
import { performance } from 'perf_hooks'
const readlineSync = require('@jcbuisson/readlinesync')
const decache = require('decache')
import LineByLine from './LineByLine.js'
class JSTable extends EventEmitter {
export default class JSTable extends EventEmitter {
#data = null
#options = null
#dataProxy = null
......@@ -39,13 +39,12 @@ class JSTable extends EventEmitter {
// along with the code in the persistChange() method is commented out.
// Do not remove this as it would be a pain to have to rewrite it every
// time we need to debug it.
// #numberOfWrites = 0
// numberOfWrites = 0
// Either loads the table at the passed table path (default) or, if
// data is passed, creates a new table at table path, populating
// it with the passed data.
constructor(tablePath, data = null, options = { compactOnLoad:true, alwaysUseLineByLineLoads: false }
) {
constructor(tablePath, data = null, options = { compactOnLoad:true, alwaysUseLineByLineLoads: false }) {
super()
this.tablePath = tablePath
......@@ -72,22 +71,23 @@ class JSTable extends EventEmitter {
//
// Related information:
//
// - https://github.com/nodejs/node/issues/28513#issuecomment-699680062
// - https://github.com/nodejs/node/issues/28513issuecomment-699680062
// - https://danluu.com/file-consistency/
this.#writeStream = fs.createWriteStream(this.tablePath, {flags: 'as'})
// NB. we are returning the data proxy, not an
// instance of JSTable. Use accordingly.
// Note: coverage ignore is due to this bug in c8: https://github.com/bcoe/c8/issues/290
/* c8 ignore next 2 */
return this.#dataProxy
}
_create (tablePath = null) {
const serialisedData = JSDF.serialise(this.#data, 'globalThis._', null)
const serialisedData = JSDF.serialise(this.#data, 'export const _', null)
this.#dataProxy = DataProxy.createDeepProxy(this, this.#data, '_')
fs.appendFileSync(tablePath || this.tablePath,
`${serialisedData}(function () { if (typeof define === 'function' && define.amd) { define([], globalThis._); } else if (typeof module === 'object' && module.exports) { module.exports = globalThis._ } else { globalThis.${this.tableName} = globalThis._ } })();\n`)
fs.appendFileSync(tablePath || this.tablePath, serialisedData)
}
......@@ -105,10 +105,10 @@ class JSTable extends EventEmitter {
const t1 = performance.now()
log(` 💾 ❨JSDB❩ Compacting and persisting table ${this.tableName}…`)
const compactedFilePath = `${this.tablePath}.compacted.tmp`
fs.removeSync(compactedFilePath)
fs.rmSync(compactedFilePath, {recursive: true, force: true})
delete this.#data.__id__ // We don’t want to set the ID twice as the creation process will.
this._create(compactedFilePath)
fs.moveSync(compactedFilePath, this.tablePath, { overwrite: true })
fs.renameSync(compactedFilePath, this.tablePath, { overwrite: true })
log(` 💾 ❨JSDB❩ ╰─ Compacted and persisted table in ${(performance.now() - t1).toFixed(3)} ms.`)
}
......@@ -119,13 +119,6 @@ class JSTable extends EventEmitter {
return new Promise((resolve, reject) => {
this.#writeStream.end(() => {
// If the table was loaded via require(), this will remove it from the
// cache so that it is loaded fresh from disk on the next attempt.
// (If we don’t do this, all changes since the process started would
// be lost when the table is reloaded from cache.)
decache(this.tablePath)
log(` 💾 ❨JSDB❩ │ ╰─ Closed table ${this.tableName}.`)
resolve()
})
......@@ -137,7 +130,7 @@ class JSTable extends EventEmitter {
async delete () {
log(` 💾 ❨JSDB❩ Deleting table ${this.tableName}…`)
await this.close()
await fs.remove(this.tablePath)
await fsPromises.rm(this.tablePath, {recursive: true, force: true})
log(` 💾 ❨JSDB❩ ╰─ Table in ${this.tableName} deleted.`)
this.emit('delete', this.#dataProxy)
}
......@@ -155,36 +148,63 @@ class JSTable extends EventEmitter {
if (tableSize < LOAD_STRATEGY_CHANGE_LIMIT && !this.#options.alwaysUseLineByLineLoads) {
//
// Regular load, use require().
// Regular load, read in the file synchronously and evaluate the
// JavaScript log manually.
//
log(` 💾 ❨JSDB❩ ╰─ Loading table synchronously.`)
this.#data = require(path.resolve(this.tablePath))
let table = fs.readFileSync(this.tablePath, 'utf-8')
// Create a local constant to hold the root data structure based on the
// one exported in the first line of the table and populate it with
// the initial/compacted data provided for it on that line.
const indexOfFirstNewline = table.indexOf('\n')
const initialData = table.substr(0, indexOfFirstNewline).replace('export const ', '')
let _
eval(initialData)
// Remove the first line.
table = table.substr(indexOfFirstNewline)
// Evaluate the remaining JavaScript operations in the log.
eval(table)
// Null out the originally-loaded data structure since we don’t need it any longer.
// (The garbage collector will collect this eventually anyway.)
table = null
this.#data = _
} else {
//
// Large table load strategy.
// Large table load strategy (synchronous readline).
//
// (Note that Node.js has a 1GB hard limit on string size so no transaction in the
// table can be bigger than this or Node will crash. This should never be a problem.)
// table can be bigger than this or Node will crash. This should never be a problem.
// Also note that his loading strategy is there just to make sure we make a best
// effort to try and support larger data sets but if you are working with this
// much data, you’re going to also have to raise the various Node.js limits when
// you run into them. See the documentation for further details.
//
log(` 💾 ❨JSDB❩ ╰─ Streaming table load for large table (> 500MB).`)
log(` 💾 ❨JSDB❩ ╰─ Line-by-line table load for large table (> 500MB).`)
this.#options.compactOnLoad = false
log(` 💾 ❨JSDB❩ ╰─ Note: compaction is disabled for large tables (> 500MB) for performance reasons.`)
const lines = readlineSync(this.tablePath)
const lines = new LineByLine(this.tablePath)
//
// Since we’re running under Node, the UMD-style (https://github.com/umdjs/umd)
// IIFE (https://developer.mozilla.org/en-US/docs/Glossary/IIFE)) will execute a module.exports statement.
// Which is not what we want here. So we handle the header manually.
//
eval(lines.next().value) // Create the correct root object of the object graph and assign it to const _.
lines.next() // Skip the require() statement in the header.
// Create the correct root object of the object graph and assign it to const _.
const initialData = lines.next().toString('utf-8').replace('export const ', '')
let _
eval(initialData)
// Load in the rest of the data.
for (let line of lines) {
eval(line)
// Load in and evaluate the rest of the operations.
let line
while (line = lines.next()) {
eval(line.toString('utf-8'))
}
this.#data = _
}
log(` 💾 ❨JSDB❩ ╰─ Table loaded in ${Time.elapsed(tableTimingLabel)} ms.`)
if (this.#options.compactOnLoad) {
// Compaction recreates the transaction log using the loaded-in object graph
......@@ -208,14 +228,12 @@ class JSTable extends EventEmitter {
persistChange (change) {
// const writeTimingLabel = ++this.#numberOfWrites
// const writeTimingLabel = ++this.numberOfWrites
// Time.mark(writeTimingLabel)
this.#writeStream.write(change, () => {
// log(` 💾 ❨JSDB❩ Write #${writeTimingLabel} took ${Time.elapsed(writeTimingLabel)}`)
// log(` 💾 ❨JSDB❩ Write ${writeTimingLabel} took ${Time.elapsed(writeTimingLabel)}`)
this.emit('persist', this, change)
})
}
}
module.exports = JSTable
// The MIT License (MIT)
// Copyright (c) 2013 Liucw
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
// Inlined and converted to ESM from: n-readlines@1.0.1
// https://github.com/nacholibre/node-readlines
import fs from 'fs'
/**
* @class
*/
export default class LineByLine {
constructor(file, options) {
options = options || {};
if (!options.readChunk) options.readChunk = 1024;
if (!options.newLineCharacter) {
options.newLineCharacter = 0x0a; //linux line ending
} else {
options.newLineCharacter = options.newLineCharacter.charCodeAt(0);
}
if (typeof file === 'number') {
this.fd = file;
} else {
this.fd = fs.openSync(file, 'r');
}
this.options = options;
this.newLineCharacter = options.newLineCharacter;
this.reset();
}
_searchInBuffer(buffer, hexNeedle) {
let found = -1;
for (let i = 0; i <= buffer.length; i++) {
let b_byte = buffer[i];
if (b_byte === hexNeedle) {
found = i;
break;
}
}
return found;
}
reset() {
this.eofReached = false;
this.linesCache = [];
this.fdPosition = 0;
}
close() {
fs.closeSync(this.fd);
this.fd = null;
}
_extractLines(buffer) {
let line;
const lines = [];
let bufferPosition = 0;
let lastNewLineBufferPosition = 0;
while (true) {
let bufferPositionValue = buffer[bufferPosition++];
if (bufferPositionValue === this.newLineCharacter) {
line = buffer.slice(lastNewLineBufferPosition, bufferPosition);
lines.push(line);
lastNewLineBufferPosition = bufferPosition;
} else if (bufferPositionValue === undefined) {
break;
}
}
let leftovers = buffer.slice(lastNewLineBufferPosition, bufferPosition);
if (leftovers.length) {
lines.push(leftovers);
}
return lines;
};
_readChunk(lineLeftovers) {
let totalBytesRead = 0;
let bytesRead;
const buffers = [];
do {
const readBuffer = new Buffer.alloc(this.options.readChunk);
bytesRead = fs.readSync(this.fd, readBuffer, 0, this.options.readChunk, this.fdPosition);
totalBytesRead = totalBytesRead + bytesRead;
this.fdPosition = this.fdPosition + bytesRead;
buffers.push(readBuffer);
} while (bytesRead && this._searchInBuffer(buffers[buffers.length-1], this.options.newLineCharacter) === -1);
let bufferData = Buffer.concat(buffers);
if (bytesRead < this.options.readChunk) {
this.eofReached = true;
bufferData = bufferData.slice(0, totalBytesRead);
}