OpenServer
/
NodeLogging
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Huge performance improvement through better queue system.

This commit is contained in:
User user 2021-05-19 13:50:04 +02:00
parent ee3123f400
commit 6c65d2c83d
4 changed files with 43 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
package-lock.json generated
View File

@ -1,15 +1,15 @@
{
"name": "@hibas123/nodelogging",
"version": "3.1.2",
"version": "3.1.3",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@hibas123/nodelogging",
"version": "3.1.2",
"version": "3.1.3",
"license": "MIT",
"dependencies": {
"@hibas123/logging": "^3.1.1",
"@hibas123/logging": "^3.1.2",
"@hibas123/utils": "^2.2.18"
},
"devDependencies": {
@ -107,9 +107,9 @@
}
},
"node_modules/@hibas123/logging": {
"version": "3.1.1",
"resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.1.tgz",
"integrity": "sha512-A1zUOdczajpCURqX0JAoTeadLLSZsLBQg9dQV1DK5zVcyrLBbFqt3fkE41tvE097bm8sjjYkx9v+HSRzvqzGRg==",
"version": "3.1.2",
"resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.2.tgz",
"integrity": "sha512-zGtKxcb1FIJvB3RSTdkGMqVUmxKORlXutS3rrx/KpjAec7vFHsTqNilp+QkXiHijFDM4PiyI/D31tzWXgp0UJw==",
"license": "MIT"
},
"node_modules/@hibas123/utils": {
@ -2179,9 +2179,9 @@
}
},
"@hibas123/logging": {
"version": "3.1.1",
"resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.1.tgz",
"integrity": "sha512-A1zUOdczajpCURqX0JAoTeadLLSZsLBQg9dQV1DK5zVcyrLBbFqt3fkE41tvE097bm8sjjYkx9v+HSRzvqzGRg=="
"version": "3.1.2",
"resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.2.tgz",
"integrity": "sha512-zGtKxcb1FIJvB3RSTdkGMqVUmxKORlXutS3rrx/KpjAec7vFHsTqNilp+QkXiHijFDM4PiyI/D31tzWXgp0UJw=="
},
"@hibas123/utils": {
"version": "2.2.18",

4
package.json
View File

@ -1,6 +1,6 @@
{
"name": "@hibas123/nodelogging",
"version": "3.1.2",
"version": "3.1.3",
"description": "",
"main": "out/index.js",
"types": "out/index.d.ts",
@ -33,7 +33,7 @@
"typescript": "^4.2.4"
},
"dependencies": {
"@hibas123/logging": "^3.1.1",
"@hibas123/logging": "^3.1.2",
"@hibas123/utils": "^2.2.18"
}
}

13
src/benchmark.ts
View File

@ -30,13 +30,13 @@ async function benchmark(
Promise.resolve().then(async () => {
const largeText = "hallowelt!".repeat(250);
const lg = new LoggingBase({
console: false,
});
const fs = new FileAdapter("logs/benchmark", Number.MAX_SAFE_INTEGER);
await lg.addAdapter(fs);
await benchmark("large data", 100000, async (cnt) => {
const lg = new LoggingBase({
console: false,
});
const fs = new FileAdapter("logs/benchmark", Number.MAX_SAFE_INTEGER);
await lg.addAdapter(fs);
console.time("Logging");
for (let i = 0; i < cnt; i++) {
lg.log(largeText);
@ -46,5 +46,6 @@ Promise.resolve().then(async () => {
await fs.close();
await lg.close();
});
console.table(results);
});

45
src/filewriter.ts
View File

@ -67,6 +67,8 @@ const Debounce = (callback: () => void, iv = 500, max = 100) => {
};
};
const QUEUE_START_SIZE = 10000;
export class Files {
private static files = new Map<string, Files>();
static getFile(filename: string): Files {
@ -88,7 +90,8 @@ export class Files {
#lock = new Lock();
#debounce = Debounce(this.checkQueue.bind(this));
#initialized = false;
#queue: Buffer[] = [];
#queue: Buffer[] = new Array(QUEUE_START_SIZE);
#queueIdx = 0;
public get initlialized() {
return this.#initialized;
@ -116,7 +119,6 @@ export class Files {
}
private async initializeFile(new_file = false) {
console.time("init");
try {
if (this.#stream) {
const closePrms = once(this.#stream, "close");
@ -140,36 +142,41 @@ export class Files {
this.#stream = fs.createWriteStream(this.file, { flags: "a" });
this.#size = size;
} catch (e) {
console.log(e);
} catch (err) {
console.log(err);
//TODO: is this the right behavior? Probably not...
process.exit(1);
}
console.timeEnd("init");
}
private async checkQueue(nolock: boolean = false) {
let lock: any;
if (nolock == false) {
//TODO: New design might cause new messages to be "stalled" till close or another message
if (this.#lock.locked) return;
lock = await this.#lock.getLock();
}
let entry: Buffer;
console.log("Check queue with", this.#queue.length);
let c = 0;
const queue = this.#queue;
const queueCnt = this.#queueIdx;
this.#queue = new Array(QUEUE_START_SIZE);
this.#queueIdx = 0;
let buffer = Buffer.alloc(1024 * 128);
let ci = 0;
while ((entry = this.#queue.shift())) {
if (entry.length > buffer.length) {
this.write_to_file(entry.slice(0, ci));
for (let i = 0; i < queueCnt; i++) {
const entry = queue[i];
if (entry.length + ci > buffer.length) {
await this.write_to_file(buffer.slice(0, ci));
ci = 0;
this.write_to_file(entry);
} else if (entry.length + ci > buffer.length) {
this.write_to_file(entry.slice(0, ci));
ci = 0;
entry.copy(buffer, ci);
ci += entry.length;
if (entry.length > buffer.length) {
await this.write_to_file(entry);
} else {
entry.copy(buffer, ci);
ci += entry.length;
}
} else {
entry.copy(buffer, ci);
ci += entry.length;
@ -180,8 +187,6 @@ export class Files {
await this.write_to_file(buffer.slice(0, ci));
}
console.log("Check queue real", c);
if (lock) lock.release();
}
@ -218,7 +223,7 @@ export class Files {
}
public write(data: Buffer) {
this.#queue.push(data);
this.#queue[this.#queueIdx++] = data;
this.#debounce.trigger();
}
}