mirror of
https://git.stamm.me/OpenServer/NodeLogging.git
synced 2024-11-21 12:49:24 +00:00
Huge performance improvement through better queue system.
This commit is contained in:
parent
ee3123f400
commit
6c65d2c83d
18
package-lock.json
generated
18
package-lock.json
generated
@ -1,15 +1,15 @@
|
||||
{
|
||||
"name": "@hibas123/nodelogging",
|
||||
"version": "3.1.2",
|
||||
"version": "3.1.3",
|
||||
"lockfileVersion": 2,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@hibas123/nodelogging",
|
||||
"version": "3.1.2",
|
||||
"version": "3.1.3",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@hibas123/logging": "^3.1.1",
|
||||
"@hibas123/logging": "^3.1.2",
|
||||
"@hibas123/utils": "^2.2.18"
|
||||
},
|
||||
"devDependencies": {
|
||||
@ -107,9 +107,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@hibas123/logging": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.1.tgz",
|
||||
"integrity": "sha512-A1zUOdczajpCURqX0JAoTeadLLSZsLBQg9dQV1DK5zVcyrLBbFqt3fkE41tvE097bm8sjjYkx9v+HSRzvqzGRg==",
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.2.tgz",
|
||||
"integrity": "sha512-zGtKxcb1FIJvB3RSTdkGMqVUmxKORlXutS3rrx/KpjAec7vFHsTqNilp+QkXiHijFDM4PiyI/D31tzWXgp0UJw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@hibas123/utils": {
|
||||
@ -2179,9 +2179,9 @@
|
||||
}
|
||||
},
|
||||
"@hibas123/logging": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.1.tgz",
|
||||
"integrity": "sha512-A1zUOdczajpCURqX0JAoTeadLLSZsLBQg9dQV1DK5zVcyrLBbFqt3fkE41tvE097bm8sjjYkx9v+HSRzvqzGRg=="
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.2.tgz",
|
||||
"integrity": "sha512-zGtKxcb1FIJvB3RSTdkGMqVUmxKORlXutS3rrx/KpjAec7vFHsTqNilp+QkXiHijFDM4PiyI/D31tzWXgp0UJw=="
|
||||
},
|
||||
"@hibas123/utils": {
|
||||
"version": "2.2.18",
|
||||
|
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@hibas123/nodelogging",
|
||||
"version": "3.1.2",
|
||||
"version": "3.1.3",
|
||||
"description": "",
|
||||
"main": "out/index.js",
|
||||
"types": "out/index.d.ts",
|
||||
@ -33,7 +33,7 @@
|
||||
"typescript": "^4.2.4"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hibas123/logging": "^3.1.1",
|
||||
"@hibas123/logging": "^3.1.2",
|
||||
"@hibas123/utils": "^2.2.18"
|
||||
}
|
||||
}
|
||||
|
@ -30,13 +30,13 @@ async function benchmark(
|
||||
Promise.resolve().then(async () => {
|
||||
const largeText = "hallowelt!".repeat(250);
|
||||
|
||||
const lg = new LoggingBase({
|
||||
console: false,
|
||||
});
|
||||
const fs = new FileAdapter("logs/benchmark", Number.MAX_SAFE_INTEGER);
|
||||
await lg.addAdapter(fs);
|
||||
|
||||
await benchmark("large data", 100000, async (cnt) => {
|
||||
const lg = new LoggingBase({
|
||||
console: false,
|
||||
});
|
||||
const fs = new FileAdapter("logs/benchmark", Number.MAX_SAFE_INTEGER);
|
||||
await lg.addAdapter(fs);
|
||||
|
||||
console.time("Logging");
|
||||
for (let i = 0; i < cnt; i++) {
|
||||
lg.log(largeText);
|
||||
@ -46,5 +46,6 @@ Promise.resolve().then(async () => {
|
||||
await fs.close();
|
||||
await lg.close();
|
||||
});
|
||||
|
||||
console.table(results);
|
||||
});
|
||||
|
@ -67,6 +67,8 @@ const Debounce = (callback: () => void, iv = 500, max = 100) => {
|
||||
};
|
||||
};
|
||||
|
||||
const QUEUE_START_SIZE = 10000;
|
||||
|
||||
export class Files {
|
||||
private static files = new Map<string, Files>();
|
||||
static getFile(filename: string): Files {
|
||||
@ -88,7 +90,8 @@ export class Files {
|
||||
#lock = new Lock();
|
||||
#debounce = Debounce(this.checkQueue.bind(this));
|
||||
#initialized = false;
|
||||
#queue: Buffer[] = [];
|
||||
#queue: Buffer[] = new Array(QUEUE_START_SIZE);
|
||||
#queueIdx = 0;
|
||||
|
||||
public get initlialized() {
|
||||
return this.#initialized;
|
||||
@ -116,7 +119,6 @@ export class Files {
|
||||
}
|
||||
|
||||
private async initializeFile(new_file = false) {
|
||||
console.time("init");
|
||||
try {
|
||||
if (this.#stream) {
|
||||
const closePrms = once(this.#stream, "close");
|
||||
@ -140,36 +142,41 @@ export class Files {
|
||||
|
||||
this.#stream = fs.createWriteStream(this.file, { flags: "a" });
|
||||
this.#size = size;
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
} catch (err) {
|
||||
console.log(err);
|
||||
//TODO: is this the right behavior? Probably not...
|
||||
process.exit(1);
|
||||
}
|
||||
console.timeEnd("init");
|
||||
}
|
||||
|
||||
private async checkQueue(nolock: boolean = false) {
|
||||
let lock: any;
|
||||
if (nolock == false) {
|
||||
//TODO: New design might cause new messages to be "stalled" till close or another message
|
||||
if (this.#lock.locked) return;
|
||||
lock = await this.#lock.getLock();
|
||||
}
|
||||
|
||||
let entry: Buffer;
|
||||
console.log("Check queue with", this.#queue.length);
|
||||
let c = 0;
|
||||
const queue = this.#queue;
|
||||
const queueCnt = this.#queueIdx;
|
||||
|
||||
this.#queue = new Array(QUEUE_START_SIZE);
|
||||
this.#queueIdx = 0;
|
||||
|
||||
let buffer = Buffer.alloc(1024 * 128);
|
||||
let ci = 0;
|
||||
while ((entry = this.#queue.shift())) {
|
||||
if (entry.length > buffer.length) {
|
||||
this.write_to_file(entry.slice(0, ci));
|
||||
for (let i = 0; i < queueCnt; i++) {
|
||||
const entry = queue[i];
|
||||
|
||||
if (entry.length + ci > buffer.length) {
|
||||
await this.write_to_file(buffer.slice(0, ci));
|
||||
ci = 0;
|
||||
this.write_to_file(entry);
|
||||
} else if (entry.length + ci > buffer.length) {
|
||||
this.write_to_file(entry.slice(0, ci));
|
||||
ci = 0;
|
||||
entry.copy(buffer, ci);
|
||||
ci += entry.length;
|
||||
if (entry.length > buffer.length) {
|
||||
await this.write_to_file(entry);
|
||||
} else {
|
||||
entry.copy(buffer, ci);
|
||||
ci += entry.length;
|
||||
}
|
||||
} else {
|
||||
entry.copy(buffer, ci);
|
||||
ci += entry.length;
|
||||
@ -180,8 +187,6 @@ export class Files {
|
||||
await this.write_to_file(buffer.slice(0, ci));
|
||||
}
|
||||
|
||||
console.log("Check queue real", c);
|
||||
|
||||
if (lock) lock.release();
|
||||
}
|
||||
|
||||
@ -218,7 +223,7 @@ export class Files {
|
||||
}
|
||||
|
||||
public write(data: Buffer) {
|
||||
this.#queue.push(data);
|
||||
this.#queue[this.#queueIdx++] = data;
|
||||
this.#debounce.trigger();
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user