Huge performance improvement through better queue system.

package-lock.json

@@ -1,15 +1,15 @@
 {
    "name": "@hibas123/nodelogging",
-   "version": "3.1.2",
+   "version": "3.1.3",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@hibas123/nodelogging",
-         "version": "3.1.2",
+         "version": "3.1.3",
          "license": "MIT",
          "dependencies": {
-            "@hibas123/logging": "^3.1.1",
+            "@hibas123/logging": "^3.1.2",
             "@hibas123/utils": "^2.2.18"
          },
          "devDependencies": {
@@ -107,9 +107,9 @@
          }
       },
       "node_modules/@hibas123/logging": {
-         "version": "3.1.1",
-         "resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.1.tgz",
-         "integrity": "sha512-A1zUOdczajpCURqX0JAoTeadLLSZsLBQg9dQV1DK5zVcyrLBbFqt3fkE41tvE097bm8sjjYkx9v+HSRzvqzGRg==",
+         "version": "3.1.2",
+         "resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.2.tgz",
+         "integrity": "sha512-zGtKxcb1FIJvB3RSTdkGMqVUmxKORlXutS3rrx/KpjAec7vFHsTqNilp+QkXiHijFDM4PiyI/D31tzWXgp0UJw==",
          "license": "MIT"
       },
       "node_modules/@hibas123/utils": {
@@ -2179,9 +2179,9 @@
          }
       },
       "@hibas123/logging": {
-         "version": "3.1.1",
-         "resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.1.tgz",
-         "integrity": "sha512-A1zUOdczajpCURqX0JAoTeadLLSZsLBQg9dQV1DK5zVcyrLBbFqt3fkE41tvE097bm8sjjYkx9v+HSRzvqzGRg=="
+         "version": "3.1.2",
+         "resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.2.tgz",
+         "integrity": "sha512-zGtKxcb1FIJvB3RSTdkGMqVUmxKORlXutS3rrx/KpjAec7vFHsTqNilp+QkXiHijFDM4PiyI/D31tzWXgp0UJw=="
       },
       "@hibas123/utils": {
          "version": "2.2.18",

package.json

@@ -1,6 +1,6 @@
 {
    "name": "@hibas123/nodelogging",
-   "version": "3.1.2",
+   "version": "3.1.3",
    "description": "",
    "main": "out/index.js",
    "types": "out/index.d.ts",
@@ -33,7 +33,7 @@
       "typescript": "^4.2.4"
    },
    "dependencies": {
-      "@hibas123/logging": "^3.1.1",
+      "@hibas123/logging": "^3.1.2",
       "@hibas123/utils": "^2.2.18"
    }
 }

src/benchmark.ts

@@ -30,13 +30,13 @@ async function benchmark(
 Promise.resolve().then(async () => {
    const largeText = "hallowelt!".repeat(250);
 
-   const lg = new LoggingBase({
-      console: false,
-   });
-   const fs = new FileAdapter("logs/benchmark", Number.MAX_SAFE_INTEGER);
-   await lg.addAdapter(fs);
-
    await benchmark("large data", 100000, async (cnt) => {
+      const lg = new LoggingBase({
+         console: false,
+      });
+      const fs = new FileAdapter("logs/benchmark", Number.MAX_SAFE_INTEGER);
+      await lg.addAdapter(fs);
+
       console.time("Logging");
       for (let i = 0; i < cnt; i++) {
          lg.log(largeText);
@@ -46,5 +46,6 @@ Promise.resolve().then(async () => {
       await fs.close();
       await lg.close();
    });
+
    console.table(results);
 });

src/filewriter.ts

@@ -67,6 +67,8 @@ const Debounce = (callback: () => void, iv = 500, max = 100) => {
    };
 };
 
+const QUEUE_START_SIZE = 10000;
+
 export class Files {
    private static files = new Map<string, Files>();
    static getFile(filename: string): Files {
@@ -88,7 +90,8 @@ export class Files {
    #lock = new Lock();
    #debounce = Debounce(this.checkQueue.bind(this));
    #initialized = false;
-   #queue: Buffer[] = [];
+   #queue: Buffer[] = new Array(QUEUE_START_SIZE);
+   #queueIdx = 0;
 
    public get initlialized() {
       return this.#initialized;
@@ -116,7 +119,6 @@ export class Files {
    }
 
    private async initializeFile(new_file = false) {
-      console.time("init");
       try {
          if (this.#stream) {
             const closePrms = once(this.#stream, "close");
@@ -140,36 +142,41 @@ export class Files {
 
          this.#stream = fs.createWriteStream(this.file, { flags: "a" });
          this.#size = size;
-      } catch (e) {
-         console.log(e);
+      } catch (err) {
+         console.log(err);
          //TODO: is this the right behavior? Probably not...
          process.exit(1);
       }
-      console.timeEnd("init");
    }
 
    private async checkQueue(nolock: boolean = false) {
       let lock: any;
       if (nolock == false) {
+         //TODO: New design might cause new messages to be "stalled" till close or another message
          if (this.#lock.locked) return;
          lock = await this.#lock.getLock();
       }
 
-      let entry: Buffer;
-      console.log("Check queue with", this.#queue.length);
-      let c = 0;
+      const queue = this.#queue;
+      const queueCnt = this.#queueIdx;
+
+      this.#queue = new Array(QUEUE_START_SIZE);
+      this.#queueIdx = 0;
+
       let buffer = Buffer.alloc(1024 * 128);
       let ci = 0;
-      while ((entry = this.#queue.shift())) {
-         if (entry.length > buffer.length) {
-            this.write_to_file(entry.slice(0, ci));
+      for (let i = 0; i < queueCnt; i++) {
+         const entry = queue[i];
+
+         if (entry.length + ci > buffer.length) {
+            await this.write_to_file(buffer.slice(0, ci));
             ci = 0;
-            this.write_to_file(entry);
-         } else if (entry.length + ci > buffer.length) {
-            this.write_to_file(entry.slice(0, ci));
-            ci = 0;
-            entry.copy(buffer, ci);
-            ci += entry.length;
+            if (entry.length > buffer.length) {
+               await this.write_to_file(entry);
+            } else {
+               entry.copy(buffer, ci);
+               ci += entry.length;
+            }
          } else {
             entry.copy(buffer, ci);
             ci += entry.length;
@@ -180,8 +187,6 @@ export class Files {
          await this.write_to_file(buffer.slice(0, ci));
       }
 
-      console.log("Check queue real", c);
-
       if (lock) lock.release();
    }
 
@@ -218,7 +223,7 @@ export class Files {
    }
 
    public write(data: Buffer) {
-      this.#queue.push(data);
+      this.#queue[this.#queueIdx++] = data;
       this.#debounce.trigger();
    }
 }