Verified Commit 8be2c526 authored by Aral Balkan's avatar Aral Balkan
Browse files

Update performance example and update readme with the results

parent 51506d5d
......@@ -178,6 +178,7 @@ db.people.__table__.addListener('persist', (table, change) => {
## Performance characteristics
- The time complexity of reads and writes are both O(1).
- Reads are fast (take fraction of a milisecond and are about an order of magnitude slower than direct memory reads).
- Writes are fast (in the order of a couple of miliseconds on tests on my dev machine).
......@@ -192,19 +193,19 @@ db.people.__table__.addListener('persist', (table, change) => {
## Memory Usage
__TODO: THIS SECTION NEEDS TO BE RE-WRITTEN WITH STATS FOR THE NEW STREAMING TRANSACTION LOG__
The reason JSDB is fast is because it keeps the whole database in memory. Also, to provide a transparent persistence and query API, it maintains a parallel object structure of proxies. This means that the amount of memory used will be multiples of the size of your database on disk and exhibits O(N) memory complexity.
<strike>The reason JSDB is fast is because it keeps the whole database in memory. Also, to provide a transparent persistence and query API, it maintains a parallel object structure of proxies. This means that the amount of memory used will be multiples of the size of your database on disk.
Initial load time and full table write/compaction both exhibit O(N) time complexity.
For example, using the simple performance example above, we clock:
For example, here’s just one sample from a development laptop using the simple performance example in the examples folder which creates random records around ~2KB in size each:
| Number of records | Table size on disk | Memory used |
| ----------------- | ------------------ | ----------- |
| 1,000 | 183K | 6.62MB |
| 10,000 | 1.8MB | 15.67MB |
| 100,000 | 18MB | 74.50MB |
| Number of records | Table size on disk | Memory used | Initial load time | Full table write/compaction time |
| ----------------- | ------------------ | ----------- | ----------------- | -------------------------------- |
| 1,000 | 2.5MB | 15.8MB | 41.6ms | 2.7 seconds |
| 10,000 | 25MB | 121.4MB | 380.2ms | 26 seconds |
| 100,000 | 244MB | 1.2GB | 5.5 seconds | 4.6 minutes |
</strike>
(The baseline app used about 14.6MB without any table in memory. The memory used column subtracts that from the total reported memory so as not to skew the smaller dataset results.)
## Developing
......
const JSDB = require('../..')
const dummyJSON = require('dummy-json')
const { performance } = require('perf_hooks')
const process = require('process')
const Time = require('../../lib/Time')
const fs = require('fs-extra')
const path = require('path')
const faker = require('faker')
let generate = false
let numberOfRecordsToGenerate = 10000
......@@ -24,34 +24,15 @@ let db = null
if (generate) {
console.log(`Generating ${numberOfRecordsToGenerate} dummy records. Please wait…`)
const dummyDataTemplate = `
[
{{#repeat ${1000}}}
{
"id": "{{guid}}",
"domain": "{{domain}}",
"email": "{{email}}",
"stripeId": "{{guid}}"
}
{{/repeat}}
]
`
s = performance.now()
// Create only a 1,000 unique items and repeat them as generating new items is slow
// and fizzles out before 100,000 with the module we’re using.
const aThousandUniqueRecords = dummyJSON.parse(dummyDataTemplate)
let data = []
for (let j = 0; j < numberOfRecordsToGenerate/1000; j++) {
data = data.concat(JSON.parse(aThousandUniqueRecords))
const data = []
for (let i = 0; i < numberOfRecordsToGenerate; i++) {
data.push(faker.helpers.createCard())
}
e = performance.now()
console.log(`Dummy data generation took ${e-s} ms for ${numberOfRecordsToGenerate} records.`)
console.log(`Ensuring database does not exist.`)
fs.removeSync(path.resolve('./db'))
db = new JSDB('db')
db = new JSDB('db', { deleteIfExists: true })
s = performance.now()
db.accounts = data
......@@ -71,24 +52,21 @@ for (let i = 0; i < db.accounts.length; i++) {
}
console.log(`Gets took on average (${db.accounts.length} tries): ${(timings.reduce((p, c) => p + c, 0)/db.accounts.length).toFixed(5)}`)
console.log(`The name on account #${db.accounts.length/2} is: ${db.accounts[db.accounts.length/2].name}`)
console.log('\n=== Testing property change. ===\n')
s = performance.now()
db.accounts[db.accounts.length/2].domain = 'laurakalbag.com'
db.accounts[db.accounts.length/2].name = 'Laura Kalbag'
e = performance.now()
console.log(`Updating a single field in ${db.accounts.length} records took ${e-s} ms.`)
console.log('1 >>>', db.accounts[db.accounts.length/2].domain)
setTimeout(() => {
console.log('<<<<< UPDATING THE DATABASE AGAIN, DURING A WRITE :) >>>>>>')
s = performance.now()
db.accounts[db.accounts.length/2].domain = 'ar.al'
e = performance.now()
console.log(`Updating a field took ${e-s} ms for ${db.accounts.length} records.`)
console.log('2 >>>', db.accounts[db.accounts.length/2].domain)
}, 100)
console.log('Updating table again, immediately.')
s = performance.now()
db.accounts[db.accounts.length/2].name = 'Aral Balkan'
e = performance.now()
console.log(`Updating table again took ${e-s} ms for ${db.accounts.length} records.`)
const used = process.memoryUsage().heapUsed / 1024 / 1024;
console.log(`The script uses approximately ${Math.round(used * 100) / 100} MB`);
......@@ -4,77 +4,10 @@
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"bignumber.js": {
"version": "8.1.1",
"resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-8.1.1.tgz",
"integrity": "sha512-QD46ppGintwPGuL1KqmwhR0O+N2cZUg8JG/VzwI2e28sM9TqHjQB10lI4QAaMHVbLzwVLLAwEglpKPViWX+5NQ=="
},
"dummy-json": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/dummy-json/-/dummy-json-3.0.1.tgz",
"integrity": "sha512-0ed4nDwr4aiCELX1mfbAYbX2YdWPaYs30fS2dFaYGF/vBM2Ocnfmz+QhoOLk8gFQei6tibwqrsM7PmQnpir9ng==",
"requires": {
"fecha": "^3.0.3",
"handlebars": "^4.7.6",
"numbro": "^2.3.1",
"seedrandom": "^3.0.5"
}
},
"fecha": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/fecha/-/fecha-3.0.3.tgz",
"integrity": "sha512-6LQK/1jud/FZnfEEZJ7y81vw7ge81DNd/XEsX0hgMUjhS+QMljkb1C0czBaP7dMNRVrd5mw/J2J7qI2Nw+TWZw=="
},
"handlebars": {
"version": "4.7.6",
"resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.6.tgz",
"integrity": "sha512-1f2BACcBfiwAfStCKZNrUCgqNZkGsAT7UM3kkYtXuLo0KnaVfjKOyf7PRzB6++aK9STyT1Pd2ZCPe3EGOXleXA==",
"requires": {
"minimist": "^1.2.5",
"neo-async": "^2.6.0",
"source-map": "^0.6.1",
"uglify-js": "^3.1.4",
"wordwrap": "^1.0.0"
}
},
"minimist": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
},
"neo-async": {
"version": "2.6.2",
"resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz",
"integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw=="
},
"numbro": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/numbro/-/numbro-2.3.1.tgz",
"integrity": "sha512-oVGNn/Ssvz4i9C76C07NDk3QXjj4DApD5QZsJSeGckzs4YxwyWeSu4bD+s9M3BzemqN3y0DW1Fthxlb2p/+U5w==",
"requires": {
"bignumber.js": "^8.1.1"
}
},
"seedrandom": {
"version": "3.0.5",
"resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz",
"integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg=="
},
"source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
},
"uglify-js": {
"version": "3.10.4",
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.10.4.tgz",
"integrity": "sha512-kBFT3U4Dcj4/pJ52vfjCSfyLyvG9VYYuGYPmrPvAxRw/i7xHiT4VvCev+uiEMcEEiu6UNB6KgWmGtSUYIWScbw==",
"optional": true
},
"wordwrap": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz",
"integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus="
"faker": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/faker/-/faker-5.1.0.tgz",
"integrity": "sha512-RrWKFSSA/aNLP0g3o2WW1Zez7/MnMr7xkiZmoCfAGZmdkDQZ6l2KtuXHN5XjdvpRjDl8+3vf+Rrtl06Z352+Mw=="
}
}
}
......@@ -9,6 +9,6 @@
"author": "Aral Balkan",
"license": "AGPL-3.0-or-later",
"dependencies": {
"dummy-json": "^3.0.1"
"faker": "^5.1.0"
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment