Export module entrypoint

.gitattributes

@@ -0,0 +1,4 @@
+/.yarn/**            linguist-vendored
+/.yarn/releases/*    binary
+/.yarn/plugins/**/*  binary
+/.pnp.*              binary linguist-generated

.gitignore

@@ -2,4 +2,7 @@ node_modules/
 logs/
 yarn.lock
 out/
+esm/
 .history/
+.yarn/cache
+.yarn/install-state.gz

.yarnrc.yml

@@ -0,0 +1,11 @@
+nodeLinker: node-modules
+
+npmScopes:
+   "hibas123":
+      npmRegistryServer: "https://git.hibas.dev/api/packages/hibas123/npm/"
+
+plugins:
+   - path: .yarn/plugins/@yarnpkg/plugin-interactive-tools.cjs
+     spec: "@yarnpkg/plugin-interactive-tools"
+
+yarnPath: .yarn/releases/yarn-3.6.4.cjs

package-lock.json

@@ -1,20 +1,20 @@
 {
    "name": "@hibas123/nodelogging",
-   "version": "3.0.1",
+   "version": "3.1.3",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
       "": {
          "name": "@hibas123/nodelogging",
-         "version": "3.0.1",
+         "version": "3.1.3",
          "license": "MIT",
          "dependencies": {
-            "@hibas123/logging": "^3.0.3",
+            "@hibas123/logging": "^3.1.2",
             "@hibas123/utils": "^2.2.18"
          },
          "devDependencies": {
-            "@types/node": "^15.0.2",
+            "@types/node": "^15.3.0",
-            "concurrently": "^6.0.2",
+            "concurrently": "^6.1.0",
             "nodemon": "^2.0.7",
             "typescript": "^4.2.4"
          }
@@ -107,9 +107,9 @@
          }
       },
       "node_modules/@hibas123/logging": {
-         "version": "3.0.3",
+         "version": "3.1.2",
-         "resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.0.3.tgz",
+         "resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.2.tgz",
-         "integrity": "sha512-lFAQZ4N2fg/SMjcEcbcKqmkGuOgrTn+qVYvd2UEUoS28niDcvSzKdh6MpHfg7Ier0PhdPPHESTh7SSR+hRZEiw==",
+         "integrity": "sha512-zGtKxcb1FIJvB3RSTdkGMqVUmxKORlXutS3rrx/KpjAec7vFHsTqNilp+QkXiHijFDM4PiyI/D31tzWXgp0UJw==",
          "license": "MIT"
       },
       "node_modules/@hibas123/utils": {
@@ -140,9 +140,9 @@
          }
       },
       "node_modules/@types/node": {
-         "version": "15.0.2",
+         "version": "15.3.0",
-         "resolved": "https://npm.hibas123.de/@types%2fnode/-/node-15.0.2.tgz",
+         "resolved": "https://npm.hibas123.de/@types%2fnode/-/node-15.3.0.tgz",
-         "integrity": "sha512-p68+a+KoxpoB47015IeYZYRrdqMUcpbK8re/zpFB8Ld46LHC1lPEbp3EXgkEhAYEcPvjJF6ZO+869SQ0aH1dcA==",
+         "integrity": "sha512-8/bnjSZD86ZfpBsDlCIkNXIvm+h6wi9g7IqL+kmFkQ+Wvu3JrasgLElfiPgoo8V8vVfnEi0QVS12gbl94h9YsQ==",
          "dev": true,
          "license": "MIT"
       },
@@ -573,9 +573,9 @@
          "dev": true
       },
       "node_modules/concurrently": {
-         "version": "6.0.2",
+         "version": "6.1.0",
-         "resolved": "https://npm.hibas123.de/concurrently/-/concurrently-6.0.2.tgz",
+         "resolved": "https://npm.hibas123.de/concurrently/-/concurrently-6.1.0.tgz",
-         "integrity": "sha512-u+1Q0dJG5BidgUTpz9CU16yoHTt/oApFDQ3mbvHwSDgMjU7aGqy0q8ZQyaZyaNxdwRKTD872Ux3Twc6//sWA+Q==",
+         "integrity": "sha512-jy+xj49pvqeKjc2TAVXRIhrgPG51eBKDZti0kZ41kaWk9iLbyWBjH6KMFpW7peOLkEymD+ZM83Lx6UEy3N/M9g==",
          "dev": true,
          "license": "MIT",
          "dependencies": {
@@ -2179,9 +2179,9 @@
          }
       },
       "@hibas123/logging": {
-         "version": "3.0.3",
+         "version": "3.1.2",
-         "resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.0.3.tgz",
+         "resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.2.tgz",
-         "integrity": "sha512-lFAQZ4N2fg/SMjcEcbcKqmkGuOgrTn+qVYvd2UEUoS28niDcvSzKdh6MpHfg7Ier0PhdPPHESTh7SSR+hRZEiw=="
+         "integrity": "sha512-zGtKxcb1FIJvB3RSTdkGMqVUmxKORlXutS3rrx/KpjAec7vFHsTqNilp+QkXiHijFDM4PiyI/D31tzWXgp0UJw=="
       },
       "@hibas123/utils": {
          "version": "2.2.18",
@@ -2204,9 +2204,9 @@
          }
       },
       "@types/node": {
-         "version": "15.0.2",
+         "version": "15.3.0",
-         "resolved": "https://npm.hibas123.de/@types%2fnode/-/node-15.0.2.tgz",
+         "resolved": "https://npm.hibas123.de/@types%2fnode/-/node-15.3.0.tgz",
-         "integrity": "sha512-p68+a+KoxpoB47015IeYZYRrdqMUcpbK8re/zpFB8Ld46LHC1lPEbp3EXgkEhAYEcPvjJF6ZO+869SQ0aH1dcA==",
+         "integrity": "sha512-8/bnjSZD86ZfpBsDlCIkNXIvm+h6wi9g7IqL+kmFkQ+Wvu3JrasgLElfiPgoo8V8vVfnEi0QVS12gbl94h9YsQ==",
          "dev": true
       },
       "@types/normalize-package-data": {
@@ -2542,9 +2542,9 @@
          "dev": true
       },
       "concurrently": {
-         "version": "6.0.2",
+         "version": "6.1.0",
-         "resolved": "https://npm.hibas123.de/concurrently/-/concurrently-6.0.2.tgz",
+         "resolved": "https://npm.hibas123.de/concurrently/-/concurrently-6.1.0.tgz",
-         "integrity": "sha512-u+1Q0dJG5BidgUTpz9CU16yoHTt/oApFDQ3mbvHwSDgMjU7aGqy0q8ZQyaZyaNxdwRKTD872Ux3Twc6//sWA+Q==",
+         "integrity": "sha512-jy+xj49pvqeKjc2TAVXRIhrgPG51eBKDZti0kZ41kaWk9iLbyWBjH6KMFpW7peOLkEymD+ZM83Lx6UEy3N/M9g==",
          "dev": true,
          "requires": {
             "chalk": "^4.1.0",

package.json

@@ -1,16 +1,19 @@
 {
    "name": "@hibas123/nodelogging",
-   "version": "3.0.6",
+   "packageManager": "yarn@3.6.4",
+   "version": "3.1.6",
    "description": "",
    "main": "out/index.js",
    "types": "out/index.d.ts",
+   "module": "esm/index.js",
    "scripts": {
       "prepublish": "npm run build",
-      "build": "tsc",
+      "build": "tsc && tsc -p tsconfig.esm.json",
       "watch-ts": "tsc --watch",
       "watch-js": "nodemon out/test.js",
       "watch": "concurrently npm:watch-*",
       "test": "npm run build && node out/test.js",
+      "benchmark": "npm run build && node out/benchmark.js",
       "live": "nodemon out/test.js"
    },
    "repository": {
@@ -22,17 +25,18 @@
    "files": [
       "src/",
       "out/",
+      "esm/",
       "tsconfig.json",
       "readme.md"
    ],
    "devDependencies": {
-      "@types/node": "^15.0.2",
+      "@types/node": "^20.8.6",
-      "concurrently": "^6.0.2",
+      "concurrently": "^8.2.1",
-      "nodemon": "^2.0.7",
+      "nodemon": "^3.0.1",
-      "typescript": "^4.2.4"
+      "typescript": "^5.2.2"
    },
    "dependencies": {
-      "@hibas123/logging": "^3.0.6",
+      "@hibas123/logging": "^3.1.2",
       "@hibas123/utils": "^2.2.18"
    }
 }

src/benchmark.ts

@@ -0,0 +1,51 @@
+import { Formatted, LoggingBase, LoggingTypes } from "@hibas123/logging";
+import { once } from "events";
+import { createWriteStream } from "fs";
+import { FileAdapter } from "./filewriter";
+
+let results = {};
+
+async function benchmark(
+   name: string,
+   count: number,
+   runner: (cnt: number) => Promise<void>
+) {
+   console.log("Benchmark starting:", name);
+   const start = process.hrtime.bigint();
+
+   await runner(count);
+
+   const diffNS = process.hrtime.bigint() - start;
+   const diffMS = Number(diffNS / BigInt(1000 * 1000));
+
+   console.log("Benchmark ended:", name);
+
+   results[name] = {
+      count,
+      time: diffMS,
+      timePerI: (diffMS / count).toFixed(4),
+   };
+}
+
+Promise.resolve().then(async () => {
+   const largeText = "hallowelt!".repeat(250);
+
+   await benchmark("large data", 100000, async (cnt) => {
+      const lg = new LoggingBase({
+         console: false,
+      });
+      const fs = new FileAdapter("logs/benchmark", Number.MAX_SAFE_INTEGER);
+      await lg.addAdapter(fs);
+
+      console.time("Logging");
+      for (let i = 0; i < cnt; i++) {
+         lg.log(largeText);
+      }
+      console.timeEnd("Logging");
+
+      await fs.close();
+      await lg.close();
+   });
+
+   console.table(results);
+});

src/filewriter.ts

@@ -1,23 +1,33 @@
-import { Lock } from "@hibas123/utils";
+import { AwaitStore, Lock } from "@hibas123/utils";
 import * as fs from "fs";
 import * as path from "path";
-import { Adapter, Message, Formatted } from "@hibas123/logging";
+import { Adapter, Message, Formatted, LoggingTypes } from "@hibas123/logging";
+import { once } from "events";
 
 const MAX_FILE_SIZE = 500000000;
 
-export class LoggingFiles implements Adapter {
+export class FileAdapter implements Adapter {
+   level = LoggingTypes.Debug;
+
    file: Files;
+   isInit = new AwaitStore(false);
    constructor(private filename: string, private maxFileSize = MAX_FILE_SIZE) {}
 
-   init() {
+   setLevel(level: LoggingTypes) {
+      this.level = level;
+   }
+
+   async init() {
       if (!this.file) {
          this.file = Files.getFile(this.filename);
-         this.file.init(this.maxFileSize);
+         await this.file
+            .init(this.maxFileSize)
+            .then(() => this.isInit.send(true));
       }
    }
 
    flush(sync: boolean) {
-      this.file.flush(sync);
+      // return this.file.flush(sync);
    }
 
    onMessage(message: Message) {
@@ -25,22 +35,31 @@ export class LoggingFiles implements Adapter {
       this.file.write(msg);
    }
 
-   close() {
+   async close() {
-      this.file.close();
+      if (this.file) {
+         await this.file.close();
          this.file = undefined;
       }
+      this.isInit.send(false);
+   }
 }
 
 //TODO: Optimise write path
 
-const Debounce = (callback: () => void, iv = 500) => {
+const Debounce = (callback: () => void, iv = 500, max = 100) => {
    let to: any;
+   let curr = 0;
 
    return {
       trigger: () => {
-         if (!to) {
+         curr++;
+         if (curr >= max) {
+            curr = 0; // not clearing timeout, since this is a very high cost operation
+            callback();
+         } else if (!to) {
             to = setTimeout(() => {
                to = undefined;
+               curr = 0;
                callback();
             }, iv);
          }
@@ -48,9 +67,9 @@ const Debounce = (callback: () => void, iv = 500) => {
    };
 };
 
-export class Files {
+const QUEUE_START_SIZE = 10000;
-   private open = 0;
 
+export class Files {
    private static files = new Map<string, Files>();
    static getFile(filename: string): Files {
       filename = path.resolve(filename);
@@ -63,13 +82,16 @@ export class Files {
       return file;
    }
 
-   private maxFileSize = MAX_FILE_SIZE;
+   private open = 0;
-   private size: number = 0;
-   private stream: fs.WriteStream = undefined;
-   private lock = new Lock();
-   private debounce = Debounce(() => this.checkQueue);
 
+   #maxFileSize = MAX_FILE_SIZE;
+   #size: number = 0;
+   #stream: fs.WriteStream = undefined;
+   #lock = new Lock();
+   #debounce = Debounce(this.checkQueue.bind(this));
    #initialized = false;
+   #queue: Buffer[] = new Array(QUEUE_START_SIZE);
+   #queueIdx = 0;
 
    public get initlialized() {
       return this.#initialized;
@@ -79,19 +101,10 @@ export class Files {
 
    public async init(maxFileSize: number) {
       if (this.#initialized) return;
-      let lock = await this.lock.getLock();
+      this.#maxFileSize = maxFileSize;
-      this.maxFileSize == maxFileSize;
+
-      await this.initializeFile();
+      let lock = await this.#lock.getLock();
-      this.#initialized = true;
-      lock.release();
-      this.checkQueue();
-   }
 
-   private async initializeFile(new_file = false) {
-      try {
-         if (this.stream) {
-            this.stream.close();
-         }
       const folder = path.dirname(this.file);
       if (folder) {
          if (!(await fsExists(folder))) {
@@ -99,10 +112,26 @@ export class Files {
          }
       }
 
+      await this.initializeFile();
+      this.#initialized = true;
+      await this.checkQueue(true);
+      lock.release();
+   }
+
+   private async initializeFile(new_file = false) {
+      try {
+         if (this.#stream) {
+            const closePrms = once(this.#stream, "close");
+
+            this.#stream.end();
+
+            await closePrms;
+         }
+
          let size = 0;
          if (await fsExists(this.file)) {
             let stats = await fsStat(this.file);
-            if (new_file || stats.size >= this.maxFileSize) {
+            if (new_file || stats.size >= this.#maxFileSize) {
                if (await fsExists(this.file + ".old"))
                   await fsUnlink(this.file + ".old");
                await fsMove(this.file, this.file + ".old");
@@ -111,62 +140,80 @@ export class Files {
             }
          }
 
-         this.stream = fs.createWriteStream(this.file, { flags: "a" });
+         this.#stream = fs.createWriteStream(this.file, { flags: "a" });
-         this.size = size;
+         this.#size = size;
-      } catch (e) {
+      } catch (err) {
-         console.log(e);
+         console.log(err);
-         //TODO: is this the right behavior?
+         //TODO: is this the right behavior? Probably not...
          process.exit(1);
       }
    }
 
-   private queue: Buffer[] = [];
+   private async checkQueue(nolock: boolean = false) {
-
+      let lock: any;
-   async checkQueue() {
+      if (nolock == false) {
-      if (this.lock.locked) return;
+         //TODO: New design might cause new messages to be "stalled" till close or another message
-      let lock = await this.lock.getLock();
+         if (this.#lock.locked) return;
-      let msg: Buffer;
+         lock = await this.#lock.getLock();
-      while ((msg = this.queue.shift())) {
-         await this.write_to_file(msg);
       }
-      lock.release();
+
+      const queue = this.#queue;
+      const queueCnt = this.#queueIdx;
+
+      this.#queue = new Array(QUEUE_START_SIZE);
+      this.#queueIdx = 0;
+
+      let buffer = Buffer.alloc(1024 * 128);
+      let ci = 0;
+      for (let i = 0; i < queueCnt; i++) {
+         const entry = queue[i];
+
+         if (entry.length + ci > buffer.length) {
+            await this.write_to_file(buffer.slice(0, ci));
+            ci = 0;
+            if (entry.length > buffer.length) {
+               await this.write_to_file(entry);
+            } else {
+               entry.copy(buffer, ci);
+               ci += entry.length;
+            }
+         } else {
+            entry.copy(buffer, ci);
+            ci += entry.length;
+         }
+      }
+
+      if (ci > 0) {
+         await this.write_to_file(buffer.slice(0, ci));
+      }
+
+      if (lock) lock.release();
    }
 
    public async close() {
-      await this.flush(false);
+      //TODO: maybe some raceconditions when open collides with close
+      const lock = await this.#lock.getLock();
+      await this.checkQueue(true);
       this.open--;
       if (this.open <= 0) {
-         this.stream.close();
+         const a = once(this.#stream, "close");
+         this.#stream.close();
+         await a;
          Files.files.delete(this.file);
       }
-   }
-
-   public flush(sync: boolean) {
-      if (sync) {
-         // if sync flush, the process most likely is in failstate, so checkQueue stopped its work.
-         let msg: Buffer;
-         while ((msg = this.queue.shift())) {
-            this.stream.write(msg);
-         }
-      } else {
-         return Promise.resolve().then(async () => {
-            const lock = await this.lock.getLock();
       lock.release();
-            await this.checkQueue();
-         });
-      }
    }
 
    private async write_to_file(data: Buffer) {
       try {
          if (
-            data.byteLength < this.maxFileSize &&
+            data.byteLength < this.#maxFileSize &&
-            this.size + data.byteLength > this.maxFileSize
+            this.#size + data.byteLength > this.#maxFileSize
          ) {
             await this.initializeFile(true);
          }
-         this.size += data.byteLength;
+         this.#size += data.byteLength;
-         this.stream.write(data);
+         this.#stream.write(data);
       } catch (err) {
          // TODO: Better error handling!
          console.error(err);
@@ -176,14 +223,15 @@ export class Files {
    }
 
    public write(data: Buffer) {
-      this.queue.push(data);
+      this.#queue[this.#queueIdx++] = data;
-      this.debounce.trigger();
+      this.#debounce.trigger();
    }
-
-   public dispose() {}
 }
 
-function fsUnlink(path) {
+function fsUnlink(path: string) {
+   if (fs.promises?.unlink) {
+      return fs.promises.unlink(path);
+   }
    return new Promise<void>((resolve, reject) => {
       fs.unlink(path, (err) => {
          if (err) reject(err);
@@ -193,6 +241,9 @@ function fsUnlink(path) {
 }
 
 function fsStat(path: string) {
+   if (fs.promises?.stat) {
+      return fs.promises.stat(path);
+   }
    return new Promise<fs.Stats>((resolve, reject) => {
       fs.stat(path, (err, stats) => {
          if (err) reject(err);
@@ -231,7 +282,7 @@ function fsMove(oldPath: string, newPath: string) {
 
 function fsExists(path: string) {
    return new Promise<boolean>((resolve, reject) => {
-      fs.exists(path, resolve);
+      fs.access(path, (err) => resolve(!err));
    });
 }
 

src/index.ts

@@ -1,5 +1,5 @@
-export { LoggingFiles } from "./filewriter";
+export { FileAdapter } from "./filewriter";
-import { LoggingFiles } from "./filewriter";
+import { FileAdapter } from "./filewriter";
 import { LoggingBase } from "@hibas123/logging";
 import Logging from "@hibas123/logging";
 
@@ -21,8 +21,8 @@ LoggingBase.nativeFunctions = {
    },
 };
 
-export const DefaultLoggingFile = new LoggingFiles("./logs/all.log");
+export const DefaultFileAdapter = new FileAdapter("./logs/all.log");
 
-Logging.addAdapter(DefaultLoggingFile);
+Logging.addAdapter(DefaultFileAdapter);
 
 export default Logging;

src/test.ts

@@ -1,7 +1,7 @@
 import { randomBytes } from "crypto";
 import * as fs from "fs";
 import { LoggingBase } from "@hibas123/logging";
-import Logging, { DefaultLoggingFile } from ".";
+import Logging, { DefaultFileAdapter, FileAdapter } from ".";
 
 const deleteFolderRecursive = function (path: string) {
    if (fs.existsSync(path)) {
@@ -52,19 +52,32 @@ cus2.log("Hello from custom Logger 2");
 cus22.log("Hello from custom Logger 22");
 cus2.log("Hello from custom Logger 2");
 
-const BenchmarkLogger = new LoggingBase({
+async function benchmark(count: number, message_size: number) {
+   const BenchmarkLogger = new LoggingBase({
       console: false,
       name: "bench",
-});
+   });
 
-BenchmarkLogger.addAdapter(DefaultLoggingFile);
+   if (fs.existsSync("logs/benchmark")) {
+      fs.unlinkSync("logs/benchmark");
+   }
 
-async function benchmark(count: number, message_size: number) {
+   if (fs.existsSync("logs/benchmark.old")) {
-   const randData = randomBytes(message_size).toString("hex");
+      fs.unlinkSync("logs/benchmark.old");
+   }
+
+   const BenchmarkFile = new FileAdapter("logs/benchmark");
+
+   BenchmarkLogger.addAdapter(BenchmarkFile);
+
+   const randData = randomBytes(message_size / 2).toString("hex");
    const t = process.hrtime();
    for (let i = 0; i < count; i++) {
       BenchmarkLogger.log(randData);
    }
+   await BenchmarkFile.flush(false);
+   await BenchmarkLogger.close();
+
    const diff = process.hrtime(t);
    const NS_PER_SEC = 1e9;
    const ns = diff[0] * NS_PER_SEC + diff[1];
@@ -79,12 +92,12 @@ async function benchmark(count: number, message_size: number) {
 const benchTimer = Logging.time("benchmark");
 Promise.resolve().then(async () => {
    console.log("Large data benchmark:");
-   await benchmark(7000, 50000);
+   await benchmark(70000, 50000);
 
    console.log("Realdata data benchmark:");
    await benchmark(100000, 100);
    benchTimer.end();
-});
 
-const timer = Logging.time("Test Timer");
+   const timer = Logging.time("Test Timer");
-setTimeout(() => timer.end(), 1000);
+   setTimeout(() => timer.end(), 1000);
+});

tsconfig.esm.json

@@ -0,0 +1,11 @@
+{
+   "extends": "./tsconfig.json",
+   "compilerOptions": {
+      "module": "ESNext",
+      "target": "ES2017",
+      "moduleResolution": "node",
+      "outDir": "esm"
+   },
+   "exclude": ["node_modules"],
+   "include": ["src"]
+}