OpenServer/NodeLogging
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Huge performance improvement through better queue system.

Browse Source
This commit is contained in:
User user 2021-05-19 13:50:04 +02:00
parent ee3123f400
commit 6c65d2c83d
4 changed files with 43 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
package-lock.json generated
View File

@ -1,15 +1,15 @@
{ {
"name": "@hibas123/nodelogging", "name": "@hibas123/nodelogging",
"version": "3.1.2", "version": "3.1.3",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@hibas123/nodelogging", "name": "@hibas123/nodelogging",
"version": "3.1.2", "version": "3.1.3",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@hibas123/logging": "^3.1.1", "@hibas123/logging": "^3.1.2",
"@hibas123/utils": "^2.2.18" "@hibas123/utils": "^2.2.18"
}, },
"devDependencies": { "devDependencies": {
@ -107,9 +107,9 @@
} }
}, },
"node_modules/@hibas123/logging": { "node_modules/@hibas123/logging": {
"version": "3.1.1", "version": "3.1.2",
"resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.1.tgz", "resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.2.tgz",
"integrity": "sha512-A1zUOdczajpCURqX0JAoTeadLLSZsLBQg9dQV1DK5zVcyrLBbFqt3fkE41tvE097bm8sjjYkx9v+HSRzvqzGRg==", "integrity": "sha512-zGtKxcb1FIJvB3RSTdkGMqVUmxKORlXutS3rrx/KpjAec7vFHsTqNilp+QkXiHijFDM4PiyI/D31tzWXgp0UJw==",
"license": "MIT" "license": "MIT"
}, },
"node_modules/@hibas123/utils": { "node_modules/@hibas123/utils": {
@ -2179,9 +2179,9 @@
} }
}, },
"@hibas123/logging": { "@hibas123/logging": {
"version": "3.1.1", "version": "3.1.2",
"resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.1.tgz", "resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.2.tgz",
"integrity": "sha512-A1zUOdczajpCURqX0JAoTeadLLSZsLBQg9dQV1DK5zVcyrLBbFqt3fkE41tvE097bm8sjjYkx9v+HSRzvqzGRg==" "integrity": "sha512-zGtKxcb1FIJvB3RSTdkGMqVUmxKORlXutS3rrx/KpjAec7vFHsTqNilp+QkXiHijFDM4PiyI/D31tzWXgp0UJw=="
}, },
"@hibas123/utils": { "@hibas123/utils": {
"version": "2.2.18", "version": "2.2.18",

4
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@hibas123/nodelogging", "name": "@hibas123/nodelogging",
"version": "3.1.2", "version": "3.1.3",
"description": "", "description": "",
"main": "out/index.js", "main": "out/index.js",
"types": "out/index.d.ts", "types": "out/index.d.ts",
@ -33,7 +33,7 @@
"typescript": "^4.2.4" "typescript": "^4.2.4"
}, },
"dependencies": { "dependencies": {
"@hibas123/logging": "^3.1.1", "@hibas123/logging": "^3.1.2",
"@hibas123/utils": "^2.2.18" "@hibas123/utils": "^2.2.18"
} }
} }

13
src/benchmark.ts
View File

@ -30,13 +30,13 @@ async function benchmark(
Promise.resolve().then(async () => { Promise.resolve().then(async () => {
const largeText = "hallowelt!".repeat(250); const largeText = "hallowelt!".repeat(250);
const lg = new LoggingBase({
console: false,
});
const fs = new FileAdapter("logs/benchmark", Number.MAX_SAFE_INTEGER);
await lg.addAdapter(fs);
await benchmark("large data", 100000, async (cnt) => { await benchmark("large data", 100000, async (cnt) => {
const lg = new LoggingBase({
console: false,
});
const fs = new FileAdapter("logs/benchmark", Number.MAX_SAFE_INTEGER);
await lg.addAdapter(fs);
console.time("Logging"); console.time("Logging");
for (let i = 0; i < cnt; i++) { for (let i = 0; i < cnt; i++) {
lg.log(largeText); lg.log(largeText);
@ -46,5 +46,6 @@ Promise.resolve().then(async () => {
await fs.close(); await fs.close();
await lg.close(); await lg.close();
}); });
console.table(results); console.table(results);
}); });

45
src/filewriter.ts
View File

@ -67,6 +67,8 @@ const Debounce = (callback: () => void, iv = 500, max = 100) => {
}; };
}; };
const QUEUE_START_SIZE = 10000;
export class Files { export class Files {
private static files = new Map<string, Files>(); private static files = new Map<string, Files>();
static getFile(filename: string): Files { static getFile(filename: string): Files {
@ -88,7 +90,8 @@ export class Files {
#lock = new Lock(); #lock = new Lock();
#debounce = Debounce(this.checkQueue.bind(this)); #debounce = Debounce(this.checkQueue.bind(this));
#initialized = false; #initialized = false;
#queue: Buffer[] = []; #queue: Buffer[] = new Array(QUEUE_START_SIZE);
#queueIdx = 0;
public get initlialized() { public get initlialized() {
return this.#initialized; return this.#initialized;
@ -116,7 +119,6 @@ export class Files {
} }
private async initializeFile(new_file = false) { private async initializeFile(new_file = false) {
console.time("init");
try { try {
if (this.#stream) { if (this.#stream) {
const closePrms = once(this.#stream, "close"); const closePrms = once(this.#stream, "close");
@ -140,36 +142,41 @@ export class Files {
this.#stream = fs.createWriteStream(this.file, { flags: "a" }); this.#stream = fs.createWriteStream(this.file, { flags: "a" });
this.#size = size; this.#size = size;
} catch (e) { } catch (err) {
console.log(e); console.log(err);
//TODO: is this the right behavior? Probably not... //TODO: is this the right behavior? Probably not...
process.exit(1); process.exit(1);
} }
console.timeEnd("init");
} }
private async checkQueue(nolock: boolean = false) { private async checkQueue(nolock: boolean = false) {
let lock: any; let lock: any;
if (nolock == false) { if (nolock == false) {
//TODO: New design might cause new messages to be "stalled" till close or another message
if (this.#lock.locked) return; if (this.#lock.locked) return;
lock = await this.#lock.getLock(); lock = await this.#lock.getLock();
} }
let entry: Buffer; const queue = this.#queue;
console.log("Check queue with", this.#queue.length); const queueCnt = this.#queueIdx;
let c = 0;
this.#queue = new Array(QUEUE_START_SIZE);
this.#queueIdx = 0;
let buffer = Buffer.alloc(1024 * 128); let buffer = Buffer.alloc(1024 * 128);
let ci = 0; let ci = 0;
while ((entry = this.#queue.shift())) { for (let i = 0; i < queueCnt; i++) {
if (entry.length > buffer.length) { const entry = queue[i];
this.write_to_file(entry.slice(0, ci));
if (entry.length + ci > buffer.length) {
await this.write_to_file(buffer.slice(0, ci));
ci = 0; ci = 0;
this.write_to_file(entry); if (entry.length > buffer.length) {
} else if (entry.length + ci > buffer.length) { await this.write_to_file(entry);
this.write_to_file(entry.slice(0, ci)); } else {
ci = 0; entry.copy(buffer, ci);
entry.copy(buffer, ci); ci += entry.length;
ci += entry.length; }
} else { } else {
entry.copy(buffer, ci); entry.copy(buffer, ci);
ci += entry.length; ci += entry.length;
@ -180,8 +187,6 @@ export class Files {
await this.write_to_file(buffer.slice(0, ci)); await this.write_to_file(buffer.slice(0, ci));
} }
console.log("Check queue real", c);
if (lock) lock.release(); if (lock) lock.release();
} }
@ -218,7 +223,7 @@ export class Files {
} }
public write(data: Buffer) { public write(data: Buffer) {
this.#queue.push(data); this.#queue[this.#queueIdx++] = data;
this.#debounce.trigger(); this.#debounce.trigger();
} }
} }