Verified Commit ddf1aed2 authored by Aral Balkan's avatar Aral Balkan
Browse files

Implement new JSDF serialisation for table creation/compaction

parent a16d7258
......@@ -20,7 +20,7 @@ __Needless to say, this is not ready for use yet. But feel free to take a look a
- [x] ╰─ Document queries. (1 Oct)
- [x] __Bring code coverage back up to 100%.__ (2 Oct)
- [x] __Implement safety controls on instantiation and table replacement.__ (5 Oct)
- [ ] __Implement JSDF serialiser__ (inc. support for custom objects, and Date, etc.) _(in progress)_
- [x] __Implement JSDF serialiser__ (inc. support for custom objects, and Date, etc.) (12 Oct)
- [ ] __Integrate into [Site.js](https://sitejs.org)__ _(in progress)_
- [ ] __Use/test on upcoming small-web.org site__
- [ ] __Release version 1.0.0__
......@@ -116,10 +116,8 @@ After running the above script, take a look at the resulting database table in t
JSDB tables are written into JavaScript Data Format (JSDF) files. A JSDF file is a plain JavaScript file that comprises an append-only transaction log that creates the table in memory. For our example, it looks like this:
```js
globalThis._ = [];
(function () { if (typeof define === 'function' && define.amd) { define([], globalThis._); } else if (typeof module === 'object' && module.exports) { module.exports = globalThis._ } else { globalThis.people = globalThis._ } })();
_[0] = { name: `Aral`, age: 43 };
_[1] = { name: `Laura`, age: 34 };
globalThis._ = [ { name: `Aral`, age: 43 }, { name: `Laura`, age: 34 } ];
(function () { if (typeof define === 'function' && define.amd) { define([], globalThis._); } else if (typeof mo;
_[1]['age'] = 33;
_[2] = { name: `Oskar`, age: 8 };
_[2]['name'] = `Osky`;
......@@ -152,7 +150,7 @@ Just because it’s JavaScript, it doesn’t mean that you can throw anything in
- `Number`
- `Boolean`
- `String`
- `Object`
- `Object` (and `bound Object`)
- `Array`
- `Date`
- `Symbol`
......@@ -198,10 +196,8 @@ db.people[1].introduceYourself()
If you look in the created `db/people.js` file, this time you’ll see:
```js
globalThis._ = [];
globalThis._ = [ Object.create(typeof Person === 'function' ? Person.prototype : {}, Object.getOwnPropertyDescriptors({ name: `Aral` })), Object.create(typeof Person === 'function' ? Person.prototype : {}, Object.getOwnPropertyDescriptors({ name: `Laura` })) ];
(function () { if (typeof define === 'function' && define.amd) { define([], globalThis._); } else if (typeof module === 'object' && module.exports) { module.exports = globalThis._ } else { globalThis.people = globalThis._ } })();
_[0] = Object.create(typeof Person === 'function' ? Person.prototype : {}, Object.getOwnPropertyDescriptors({ name: `Aral` }));
_[1] = Object.create(typeof Person === 'function' ? Person.prototype : {}, Object.getOwnPropertyDescriptors({ name: `Laura` }));
```
If you were to load the database in an environment where the `Person` class does not exist, you will get a regular object back.
......@@ -267,8 +263,6 @@ For details, see the [JSQL Reference](#jsql-reference) section.
## Compaction
___Note:__ I’m currently reviewing how compacting works. In a server setting, it __is__ important how fast the server restarts so I plan to make compaction a low-CPU background process that runs on a given interval instead of at every startup. This may have to be a post version 1.0 refactor._
When you load in a JSDB table, by default JSDB will compact the JSDF file.
Compaction is important for two reasons; during compaction:
......@@ -285,11 +279,8 @@ You do have the option to override the default behaviour and keep all history. Y
Now that you’ve loaded the file back, look at the `./db/people.js` JSDF file again to see how it looks after compaction:
```js
globalThis._ = [];
(function () { if (typeof define === 'function' && define.amd) { define([], globalThis._); } else if (typeof mo;
_[0] = { name: `Aral`, age: 43 };
_[1] = { name: `Laura`, age: 33 };
_[2] = { name: `Osky`, age: 8 };
globalThis._ = [ { name: `Aral`, age: 43 }, { name: `Laura`, age: 33 }, { name: `Osky`, age: 8 } ];
(function () { if (typeof define === 'function' && define.amd) { define([], globalThis._); } else if (typeof module === 'object' && module.exports) { module.exports = globalThis._ } else { globalThis.people = globalThis._ } })();
```
Ah, that is neater. You can see that Laura’s record is created with the correct age from the outset and Oskar’s name is set to its final value of Osky from the outset.
......
......@@ -40,7 +40,8 @@ class JSDF {
break
}
case 'Object': {
case 'Object':
case 'bound Object': {
const children = []
for (let childKey in value) {
const childValue = value[childKey]
......
......@@ -20,6 +20,8 @@ const isProxy = require('util').types.isProxy
const EventEmitter = require('events')
const DataProxy = require('./DataProxy')
const JSDF = require('./JSDF')
const { log, needsToBeProxified } = require('./Util')
const Time = require('./Time')
const { performance } = require('perf_hooks')
......@@ -78,29 +80,12 @@ class JSTable extends EventEmitter {
}
writeTableHeader () {
fs.appendFileSync(this.tablePath,
`globalThis._ = ${Array.isArray(this.#data) ? '[]' : '\{\}'};\n(function () { if (typeof define === 'function' && define.amd) { define([], globalThis._); } else if (typeof module === 'object' && module.exports) { module.exports = globalThis._ } else { globalThis.${this.tableName} = globalThis._ } })();\n`)
}
// This will result in the data proxy graph being
// populated from the data graph and having all necessary
// transactions for creating the data graph persisted to
// the table file on disk.
createTransactions () {
this.#dataProxy = DataProxy.createDeepProxy(this, Array.isArray(this.#data) ? [] : {}, '_')
Object.keys(this.#data).forEach(key => {
// TODO: [performance] test – would this be faster if we used locals instead of property lookup?
// Note: this assumes compaction only occurs at startup when writes are synchronous.
this.#dataProxy[key] = this.#data[key]
})
}
_create () {
this.writeTableHeader()
this.createTransactions()
const serialisedData = JSDF.serialise(this.#data, 'globalThis._', null)
this.#dataProxy = DataProxy.createDeepProxy(this, this.#data, '_')
fs.appendFileSync(this.tablePath,
`${serialisedData}(function () { if (typeof define === 'function' && define.amd) { define([], globalThis._); } else if (typeof module === 'object' && module.exports) { module.exports = globalThis._ } else { globalThis.${this.tableName} = globalThis._ } })();\n`)
}
......@@ -177,7 +162,8 @@ class JSTable extends EventEmitter {
const tableSize = fs.statSync(this.tablePath).size
if (tableSize < REQUIRE_PERFORMANCE_ADVANTAGE_SIZE_LIMIT && !this.#options.alwaysUseLineByLineLoads) {
// TODO: Need to adapt this to the new method of compaction for smaller datasets.
if (true/*tableSize < REQUIRE_PERFORMANCE_ADVANTAGE_SIZE_LIMIT && !this.#options.alwaysUseLineByLineLoads*/) {
//
// Faster to load as a module using require().
//
......
......@@ -143,10 +143,8 @@ test('basic persistence', t => {
//
const expectedTableSourceBeforeCompaction = `
globalThis._ = [];
globalThis._ = [ { name: \`aral\`, age: 44 }, { name: \`laura\`, age: 34 } ];
(function () { if (typeof define === 'function' && define.amd) { define([], globalThis._); } else if (typeof module === 'object' && module.exports) { module.exports = globalThis._ } else { globalThis.people = globalThis._ } })();
_[0] = { name: \`aral\`, age: 44 };
_[1] = { name: \`laura\`, age: 34 };
_[0]['age'] = 21;
_[0]['age'] = 43;
_[1]['age'] = 33;
......@@ -173,10 +171,8 @@ test('basic persistence', t => {
//
const expectedTableSourceAfterCompaction = `
globalThis._ = [];
globalThis._ = [ { name: \`aral\`, age: 43 }, { name: \`laura\`, age: 33 } ];
(function () { if (typeof define === 'function' && define.amd) { define([], globalThis._); } else if (typeof module === 'object' && module.exports) { module.exports = globalThis._ } else { globalThis.people = globalThis._ } })();
_[0] = { name: \`aral\`, age: 43 };
_[1] = { name: \`laura\`, age: 33 };
`
const actualTableSourceAfterCompaction = loadTableSource('db', 'people')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment