From 6c65d2c83d3ccf5292e1cb4429d20ce58c2726b3 Mon Sep 17 00:00:00 2001 From: User user Date: Wed, 19 May 2021 13:50:04 +0200 Subject: [PATCH] Huge performance improvement through better queue system. --- package-lock.json | 18 +++++++++--------- package.json | 4 ++-- src/benchmark.ts | 13 +++++++------ src/filewriter.ts | 45 +++++++++++++++++++++++++-------------------- 4 files changed, 43 insertions(+), 37 deletions(-) diff --git a/package-lock.json b/package-lock.json index 075e7f3..120b454 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,15 @@ { "name": "@hibas123/nodelogging", - "version": "3.1.2", + "version": "3.1.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@hibas123/nodelogging", - "version": "3.1.2", + "version": "3.1.3", "license": "MIT", "dependencies": { - "@hibas123/logging": "^3.1.1", + "@hibas123/logging": "^3.1.2", "@hibas123/utils": "^2.2.18" }, "devDependencies": { @@ -107,9 +107,9 @@ } }, "node_modules/@hibas123/logging": { - "version": "3.1.1", - "resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.1.tgz", - "integrity": "sha512-A1zUOdczajpCURqX0JAoTeadLLSZsLBQg9dQV1DK5zVcyrLBbFqt3fkE41tvE097bm8sjjYkx9v+HSRzvqzGRg==", + "version": "3.1.2", + "resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.2.tgz", + "integrity": "sha512-zGtKxcb1FIJvB3RSTdkGMqVUmxKORlXutS3rrx/KpjAec7vFHsTqNilp+QkXiHijFDM4PiyI/D31tzWXgp0UJw==", "license": "MIT" }, "node_modules/@hibas123/utils": { @@ -2179,9 +2179,9 @@ } }, "@hibas123/logging": { - "version": "3.1.1", - "resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.1.tgz", - "integrity": "sha512-A1zUOdczajpCURqX0JAoTeadLLSZsLBQg9dQV1DK5zVcyrLBbFqt3fkE41tvE097bm8sjjYkx9v+HSRzvqzGRg==" + "version": "3.1.2", + "resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.2.tgz", + "integrity": "sha512-zGtKxcb1FIJvB3RSTdkGMqVUmxKORlXutS3rrx/KpjAec7vFHsTqNilp+QkXiHijFDM4PiyI/D31tzWXgp0UJw==" }, "@hibas123/utils": { "version": "2.2.18", diff --git a/package.json b/package.json index 6151408..83d1a4c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hibas123/nodelogging", - "version": "3.1.2", + "version": "3.1.3", "description": "", "main": "out/index.js", "types": "out/index.d.ts", @@ -33,7 +33,7 @@ "typescript": "^4.2.4" }, "dependencies": { - "@hibas123/logging": "^3.1.1", + "@hibas123/logging": "^3.1.2", "@hibas123/utils": "^2.2.18" } } diff --git a/src/benchmark.ts b/src/benchmark.ts index b854b38..93bcb8a 100644 --- a/src/benchmark.ts +++ b/src/benchmark.ts @@ -30,13 +30,13 @@ async function benchmark( Promise.resolve().then(async () => { const largeText = "hallowelt!".repeat(250); - const lg = new LoggingBase({ - console: false, - }); - const fs = new FileAdapter("logs/benchmark", Number.MAX_SAFE_INTEGER); - await lg.addAdapter(fs); - await benchmark("large data", 100000, async (cnt) => { + const lg = new LoggingBase({ + console: false, + }); + const fs = new FileAdapter("logs/benchmark", Number.MAX_SAFE_INTEGER); + await lg.addAdapter(fs); + console.time("Logging"); for (let i = 0; i < cnt; i++) { lg.log(largeText); @@ -46,5 +46,6 @@ Promise.resolve().then(async () => { await fs.close(); await lg.close(); }); + console.table(results); }); diff --git a/src/filewriter.ts b/src/filewriter.ts index f1bc639..3603aa6 100644 --- a/src/filewriter.ts +++ b/src/filewriter.ts @@ -67,6 +67,8 @@ const Debounce = (callback: () => void, iv = 500, max = 100) => { }; }; +const QUEUE_START_SIZE = 10000; + export class Files { private static files = new Map(); static getFile(filename: string): Files { @@ -88,7 +90,8 @@ export class Files { #lock = new Lock(); #debounce = Debounce(this.checkQueue.bind(this)); #initialized = false; - #queue: Buffer[] = []; + #queue: Buffer[] = new Array(QUEUE_START_SIZE); + #queueIdx = 0; public get initlialized() { return this.#initialized; @@ -116,7 +119,6 @@ export class Files { } private async initializeFile(new_file = false) { - console.time("init"); try { if (this.#stream) { const closePrms = once(this.#stream, "close"); @@ -140,36 +142,41 @@ export class Files { this.#stream = fs.createWriteStream(this.file, { flags: "a" }); this.#size = size; - } catch (e) { - console.log(e); + } catch (err) { + console.log(err); //TODO: is this the right behavior? Probably not... process.exit(1); } - console.timeEnd("init"); } private async checkQueue(nolock: boolean = false) { let lock: any; if (nolock == false) { + //TODO: New design might cause new messages to be "stalled" till close or another message if (this.#lock.locked) return; lock = await this.#lock.getLock(); } - let entry: Buffer; - console.log("Check queue with", this.#queue.length); - let c = 0; + const queue = this.#queue; + const queueCnt = this.#queueIdx; + + this.#queue = new Array(QUEUE_START_SIZE); + this.#queueIdx = 0; + let buffer = Buffer.alloc(1024 * 128); let ci = 0; - while ((entry = this.#queue.shift())) { - if (entry.length > buffer.length) { - this.write_to_file(entry.slice(0, ci)); + for (let i = 0; i < queueCnt; i++) { + const entry = queue[i]; + + if (entry.length + ci > buffer.length) { + await this.write_to_file(buffer.slice(0, ci)); ci = 0; - this.write_to_file(entry); - } else if (entry.length + ci > buffer.length) { - this.write_to_file(entry.slice(0, ci)); - ci = 0; - entry.copy(buffer, ci); - ci += entry.length; + if (entry.length > buffer.length) { + await this.write_to_file(entry); + } else { + entry.copy(buffer, ci); + ci += entry.length; + } } else { entry.copy(buffer, ci); ci += entry.length; @@ -180,8 +187,6 @@ export class Files { await this.write_to_file(buffer.slice(0, ci)); } - console.log("Check queue real", c); - if (lock) lock.release(); } @@ -218,7 +223,7 @@ export class Files { } public write(data: Buffer) { - this.#queue.push(data); + this.#queue[this.#queueIdx++] = data; this.#debounce.trigger(); } }