OpenServer/NodeLogging
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: OpenServer:6c65d2c83d3ccf5292e1cb4429d20ce58c2726b3
Branches Tags
OpenServer:master
OpenServer:alpha
..
compare: OpenServer:95ef9238446c7e8627f9d01c1749cab77120f546
Branches Tags
OpenServer:master
OpenServer:alpha

No commits in common. "6c65d2c83d3ccf5292e1cb4429d20ce58c2726b3" and "95ef9238446c7e8627f9d01c1749cab77120f546" have entirely different histories.
6c65d2c83d ... 95ef923844

6 changed files with 113 additions and 217 deletions
Show Stats Expand all files Collapse all files

46
package-lock.json generated
View File

@ -1,20 +1,20 @@
{ {
"name": "@hibas123/nodelogging", "name": "@hibas123/nodelogging",
"version": "3.1.3", "version": "3.0.10",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@hibas123/nodelogging", "name": "@hibas123/nodelogging",
"version": "3.1.3", "version": "3.0.10",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@hibas123/logging": "^3.1.2", "@hibas123/logging": "^3.1.0",
"@hibas123/utils": "^2.2.18" "@hibas123/utils": "^2.2.18"
}, },
"devDependencies": { "devDependencies": {
"@types/node": "^15.3.0", "@types/node": "^15.0.2",
"concurrently": "^6.1.0", "concurrently": "^6.0.2",
"nodemon": "^2.0.7", "nodemon": "^2.0.7",
"typescript": "^4.2.4" "typescript": "^4.2.4"
} }
@ -107,9 +107,9 @@
} }
}, },
"node_modules/@hibas123/logging": { "node_modules/@hibas123/logging": {
"version": "3.1.2", "version": "3.1.0",
"resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.2.tgz", "resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.0.tgz",
"integrity": "sha512-zGtKxcb1FIJvB3RSTdkGMqVUmxKORlXutS3rrx/KpjAec7vFHsTqNilp+QkXiHijFDM4PiyI/D31tzWXgp0UJw==", "integrity": "sha512-iUFXdkuSYcxiIChOccTC84xldwKXJ0vdNruTiVTrU07x69BL9DvuvJ6e3CBTIy2HKgT1K1q1VChTModKPEus7A==",
"license": "MIT" "license": "MIT"
}, },
"node_modules/@hibas123/utils": { "node_modules/@hibas123/utils": {
@ -140,9 +140,9 @@
} }
}, },
"node_modules/@types/node": { "node_modules/@types/node": {
"version": "15.3.0", "version": "15.0.2",
"resolved": "https://npm.hibas123.de/@types%2fnode/-/node-15.3.0.tgz", "resolved": "https://npm.hibas123.de/@types%2fnode/-/node-15.0.2.tgz",
"integrity": "sha512-8/bnjSZD86ZfpBsDlCIkNXIvm+h6wi9g7IqL+kmFkQ+Wvu3JrasgLElfiPgoo8V8vVfnEi0QVS12gbl94h9YsQ==", "integrity": "sha512-p68+a+KoxpoB47015IeYZYRrdqMUcpbK8re/zpFB8Ld46LHC1lPEbp3EXgkEhAYEcPvjJF6ZO+869SQ0aH1dcA==",
"dev": true, "dev": true,
"license": "MIT" "license": "MIT"
}, },
@ -573,9 +573,9 @@
"dev": true "dev": true
}, },
"node_modules/concurrently": { "node_modules/concurrently": {
"version": "6.1.0", "version": "6.0.2",
"resolved": "https://npm.hibas123.de/concurrently/-/concurrently-6.1.0.tgz", "resolved": "https://npm.hibas123.de/concurrently/-/concurrently-6.0.2.tgz",
"integrity": "sha512-jy+xj49pvqeKjc2TAVXRIhrgPG51eBKDZti0kZ41kaWk9iLbyWBjH6KMFpW7peOLkEymD+ZM83Lx6UEy3N/M9g==", "integrity": "sha512-u+1Q0dJG5BidgUTpz9CU16yoHTt/oApFDQ3mbvHwSDgMjU7aGqy0q8ZQyaZyaNxdwRKTD872Ux3Twc6//sWA+Q==",
"dev": true, "dev": true,
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
@ -2179,9 +2179,9 @@
} }
}, },
"@hibas123/logging": { "@hibas123/logging": {
"version": "3.1.2", "version": "3.1.0",
"resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.2.tgz", "resolved": "https://npm.hibas123.de/@hibas123%2flogging/-/logging-3.1.0.tgz",
"integrity": "sha512-zGtKxcb1FIJvB3RSTdkGMqVUmxKORlXutS3rrx/KpjAec7vFHsTqNilp+QkXiHijFDM4PiyI/D31tzWXgp0UJw==" "integrity": "sha512-iUFXdkuSYcxiIChOccTC84xldwKXJ0vdNruTiVTrU07x69BL9DvuvJ6e3CBTIy2HKgT1K1q1VChTModKPEus7A=="
}, },
"@hibas123/utils": { "@hibas123/utils": {
"version": "2.2.18", "version": "2.2.18",
@ -2204,9 +2204,9 @@
} }
}, },
"@types/node": { "@types/node": {
"version": "15.3.0", "version": "15.0.2",
"resolved": "https://npm.hibas123.de/@types%2fnode/-/node-15.3.0.tgz", "resolved": "https://npm.hibas123.de/@types%2fnode/-/node-15.0.2.tgz",
"integrity": "sha512-8/bnjSZD86ZfpBsDlCIkNXIvm+h6wi9g7IqL+kmFkQ+Wvu3JrasgLElfiPgoo8V8vVfnEi0QVS12gbl94h9YsQ==", "integrity": "sha512-p68+a+KoxpoB47015IeYZYRrdqMUcpbK8re/zpFB8Ld46LHC1lPEbp3EXgkEhAYEcPvjJF6ZO+869SQ0aH1dcA==",
"dev": true "dev": true
}, },
"@types/normalize-package-data": { "@types/normalize-package-data": {
@ -2542,9 +2542,9 @@
"dev": true "dev": true
}, },
"concurrently": { "concurrently": {
"version": "6.1.0", "version": "6.0.2",
"resolved": "https://npm.hibas123.de/concurrently/-/concurrently-6.1.0.tgz", "resolved": "https://npm.hibas123.de/concurrently/-/concurrently-6.0.2.tgz",
"integrity": "sha512-jy+xj49pvqeKjc2TAVXRIhrgPG51eBKDZti0kZ41kaWk9iLbyWBjH6KMFpW7peOLkEymD+ZM83Lx6UEy3N/M9g==", "integrity": "sha512-u+1Q0dJG5BidgUTpz9CU16yoHTt/oApFDQ3mbvHwSDgMjU7aGqy0q8ZQyaZyaNxdwRKTD872Ux3Twc6//sWA+Q==",
"dev": true, "dev": true,
"requires": { "requires": {
"chalk": "^4.1.0", "chalk": "^4.1.0",

9
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@hibas123/nodelogging", "name": "@hibas123/nodelogging",
"version": "3.1.3", "version": "3.1.0",
"description": "", "description": "",
"main": "out/index.js", "main": "out/index.js",
"types": "out/index.d.ts", "types": "out/index.d.ts",
@ -11,7 +11,6 @@
"watch-js": "nodemon out/test.js", "watch-js": "nodemon out/test.js",
"watch": "concurrently npm:watch-*", "watch": "concurrently npm:watch-*",
"test": "npm run build && node out/test.js", "test": "npm run build && node out/test.js",
"benchmark": "npm run build && node out/benchmark.js",
"live": "nodemon out/test.js" "live": "nodemon out/test.js"
}, },
"repository": { "repository": {
@ -27,13 +26,13 @@
"readme.md" "readme.md"
], ],
"devDependencies": { "devDependencies": {
"@types/node": "^15.3.0", "@types/node": "^15.0.2",
"concurrently": "^6.1.0", "concurrently": "^6.0.2",
"nodemon": "^2.0.7", "nodemon": "^2.0.7",
"typescript": "^4.2.4" "typescript": "^4.2.4"
}, },
"dependencies": { "dependencies": {
"@hibas123/logging": "^3.1.2", "@hibas123/logging": "^3.1.0",
"@hibas123/utils": "^2.2.18" "@hibas123/utils": "^2.2.18"
} }
} }

51
src/benchmark.ts
View File

@ -1,51 +0,0 @@
import { Formatted, LoggingBase, LoggingTypes } from "@hibas123/logging";
import { once } from "events";
import { createWriteStream } from "fs";
import { FileAdapter } from "./filewriter";
let results = {};
async function benchmark(
name: string,
count: number,
runner: (cnt: number) => Promise<void>
) {
console.log("Benchmark starting:", name);
const start = process.hrtime.bigint();
await runner(count);
const diffNS = process.hrtime.bigint() - start;
const diffMS = Number(diffNS / BigInt(1000 * 1000));
console.log("Benchmark ended:", name);
results[name] = {
count,
time: diffMS,
timePerI: (diffMS / count).toFixed(4),
};
}
Promise.resolve().then(async () => {
const largeText = "hallowelt!".repeat(250);
await benchmark("large data", 100000, async (cnt) => {
const lg = new LoggingBase({
console: false,
});
const fs = new FileAdapter("logs/benchmark", Number.MAX_SAFE_INTEGER);
await lg.addAdapter(fs);
console.time("Logging");
for (let i = 0; i < cnt; i++) {
lg.log(largeText);
}
console.timeEnd("Logging");
await fs.close();
await lg.close();
});
console.table(results);
});

177
src/filewriter.ts
View File

@ -1,33 +1,29 @@
import { AwaitStore, Lock } from "@hibas123/utils"; import { Lock } from "@hibas123/utils";
import * as fs from "fs"; import * as fs from "fs";
import * as path from "path"; import * as path from "path";
import { Adapter, Message, Formatted, LoggingTypes } from "@hibas123/logging"; import { Adapter, Message, Formatted, LoggingTypes } from "@hibas123/logging";
import { once } from "events";
const MAX_FILE_SIZE = 500000000; const MAX_FILE_SIZE = 500000000;
export class FileAdapter implements Adapter { export class LoggingFiles implements Adapter {
level = LoggingTypes.Debug; level = LoggingTypes.Debug;
file: Files; file: Files;
isInit = new AwaitStore(false);
constructor(private filename: string, private maxFileSize = MAX_FILE_SIZE) {} constructor(private filename: string, private maxFileSize = MAX_FILE_SIZE) {}
setLevel(level: LoggingTypes) { setLevel(level: LoggingTypes) {
this.level = level; this.level = level;
} }
async init() { init() {
if (!this.file) { if (!this.file) {
this.file = Files.getFile(this.filename); this.file = Files.getFile(this.filename);
await this.file this.file.init(this.maxFileSize);
.init(this.maxFileSize)
.then(() => this.isInit.send(true));
} }
} }
flush(sync: boolean) { flush(sync: boolean) {
// return this.file.flush(sync); this.file.flush(sync);
} }
onMessage(message: Message) { onMessage(message: Message) {
@ -35,12 +31,9 @@ export class FileAdapter implements Adapter {
this.file.write(msg); this.file.write(msg);
} }
async close() { close() {
if (this.file) { this.file.close();
await this.file.close(); this.file = undefined;
this.file = undefined;
}
this.isInit.send(false);
} }
} }
@ -67,9 +60,9 @@ const Debounce = (callback: () => void, iv = 500, max = 100) => {
}; };
}; };
const QUEUE_START_SIZE = 10000;
export class Files { export class Files {
private open = 0;
private static files = new Map<string, Files>(); private static files = new Map<string, Files>();
static getFile(filename: string): Files { static getFile(filename: string): Files {
filename = path.resolve(filename); filename = path.resolve(filename);
@ -82,16 +75,13 @@ export class Files {
return file; return file;
} }
private open = 0; private maxFileSize = MAX_FILE_SIZE;
private size: number = 0;
private stream: fs.WriteStream = undefined;
private lock = new Lock();
private debounce = Debounce(this.checkQueue.bind(this));
#maxFileSize = MAX_FILE_SIZE;
#size: number = 0;
#stream: fs.WriteStream = undefined;
#lock = new Lock();
#debounce = Debounce(this.checkQueue.bind(this));
#initialized = false; #initialized = false;
#queue: Buffer[] = new Array(QUEUE_START_SIZE);
#queueIdx = 0;
public get initlialized() { public get initlialized() {
return this.#initialized; return this.#initialized;
@ -101,37 +91,30 @@ export class Files {
public async init(maxFileSize: number) { public async init(maxFileSize: number) {
if (this.#initialized) return; if (this.#initialized) return;
this.#maxFileSize = maxFileSize; let lock = await this.lock.getLock();
this.maxFileSize == maxFileSize;
let lock = await this.#lock.getLock();
const folder = path.dirname(this.file);
if (folder) {
if (!(await fsExists(folder))) {
await fsMkDir(folder).catch(() => {}); //Could happen, if two seperate instances want to create the same folder so ignoring
}
}
await this.initializeFile(); await this.initializeFile();
this.#initialized = true; this.#initialized = true;
await this.checkQueue(true);
lock.release(); lock.release();
this.checkQueue();
} }
private async initializeFile(new_file = false) { private async initializeFile(new_file = false) {
try { try {
if (this.#stream) { if (this.stream) {
const closePrms = once(this.#stream, "close"); this.stream.close();
}
this.#stream.end(); const folder = path.dirname(this.file);
if (folder) {
await closePrms; if (!(await fsExists(folder))) {
await fsMkDir(folder).catch(() => {}); //Could happen, if two seperate instances want to create the same folder so ignoring
}
} }
let size = 0; let size = 0;
if (await fsExists(this.file)) { if (await fsExists(this.file)) {
let stats = await fsStat(this.file); let stats = await fsStat(this.file);
if (new_file || stats.size >= this.#maxFileSize) { if (new_file || stats.size >= this.maxFileSize) {
if (await fsExists(this.file + ".old")) if (await fsExists(this.file + ".old"))
await fsUnlink(this.file + ".old"); await fsUnlink(this.file + ".old");
await fsMove(this.file, this.file + ".old"); await fsMove(this.file, this.file + ".old");
@ -140,80 +123,62 @@ export class Files {
} }
} }
this.#stream = fs.createWriteStream(this.file, { flags: "a" }); this.stream = fs.createWriteStream(this.file, { flags: "a" });
this.#size = size; this.size = size;
} catch (err) { } catch (e) {
console.log(err); console.log(e);
//TODO: is this the right behavior? Probably not... //TODO: is this the right behavior?
process.exit(1); process.exit(1);
} }
} }
private async checkQueue(nolock: boolean = false) { private queue: Buffer[] = [];
let lock: any;
if (nolock == false) { async checkQueue() {
//TODO: New design might cause new messages to be "stalled" till close or another message if (this.lock.locked) return;
if (this.#lock.locked) return; let lock = await this.lock.getLock();
lock = await this.#lock.getLock(); let msg: Buffer;
while ((msg = this.queue.shift())) {
await this.write_to_file(msg);
} }
lock.release();
const queue = this.#queue;
const queueCnt = this.#queueIdx;
this.#queue = new Array(QUEUE_START_SIZE);
this.#queueIdx = 0;
let buffer = Buffer.alloc(1024 * 128);
let ci = 0;
for (let i = 0; i < queueCnt; i++) {
const entry = queue[i];
if (entry.length + ci > buffer.length) {
await this.write_to_file(buffer.slice(0, ci));
ci = 0;
if (entry.length > buffer.length) {
await this.write_to_file(entry);
} else {
entry.copy(buffer, ci);
ci += entry.length;
}
} else {
entry.copy(buffer, ci);
ci += entry.length;
}
}
if (ci > 0) {
await this.write_to_file(buffer.slice(0, ci));
}
if (lock) lock.release();
} }
public async close() { public async close() {
//TODO: maybe some raceconditions when open collides with close await this.flush(false);
const lock = await this.#lock.getLock();
await this.checkQueue(true);
this.open--; this.open--;
if (this.open <= 0) { if (this.open <= 0) {
const a = once(this.#stream, "close"); this.stream.close();
this.#stream.close();
await a;
Files.files.delete(this.file); Files.files.delete(this.file);
} }
lock.release(); }
public flush(sync: boolean) {
if (sync) {
// if sync flush, the process most likely is in failstate, so checkQueue stopped its work.
let msg: Buffer;
while ((msg = this.queue.shift())) {
this.stream.write(msg);
}
} else {
return Promise.resolve().then(async () => {
const lock = await this.lock.getLock();
lock.release();
await this.checkQueue();
});
}
} }
private async write_to_file(data: Buffer) { private async write_to_file(data: Buffer) {
try { try {
if ( if (
data.byteLength < this.#maxFileSize && data.byteLength < this.maxFileSize &&
this.#size + data.byteLength > this.#maxFileSize this.size + data.byteLength > this.maxFileSize
) { ) {
await this.initializeFile(true); await this.initializeFile(true);
} }
this.#size += data.byteLength; this.size += data.byteLength;
this.#stream.write(data); this.stream.write(data);
} catch (err) { } catch (err) {
// TODO: Better error handling! // TODO: Better error handling!
console.error(err); console.error(err);
@ -223,15 +188,14 @@ export class Files {
} }
public write(data: Buffer) { public write(data: Buffer) {
this.#queue[this.#queueIdx++] = data; this.queue.push(data);
this.#debounce.trigger(); this.debounce.trigger();
} }
public dispose() {}
} }
function fsUnlink(path: string) { function fsUnlink(path) {
if (fs.promises?.unlink) {
return fs.promises.unlink(path);
}
return new Promise<void>((resolve, reject) => { return new Promise<void>((resolve, reject) => {
fs.unlink(path, (err) => { fs.unlink(path, (err) => {
if (err) reject(err); if (err) reject(err);
@ -241,9 +205,6 @@ function fsUnlink(path: string) {
} }
function fsStat(path: string) { function fsStat(path: string) {
if (fs.promises?.stat) {
return fs.promises.stat(path);
}
return new Promise<fs.Stats>((resolve, reject) => { return new Promise<fs.Stats>((resolve, reject) => {
fs.stat(path, (err, stats) => { fs.stat(path, (err, stats) => {
if (err) reject(err); if (err) reject(err);
@ -282,7 +243,7 @@ function fsMove(oldPath: string, newPath: string) {
function fsExists(path: string) { function fsExists(path: string) {
return new Promise<boolean>((resolve, reject) => { return new Promise<boolean>((resolve, reject) => {
fs.access(path, (err) => resolve(!err)); fs.exists(path, resolve);
}); });
} }

8
src/index.ts
View File

@ -1,5 +1,5 @@
export { FileAdapter } from "./filewriter"; export { LoggingFiles } from "./filewriter";
import { FileAdapter } from "./filewriter"; import { LoggingFiles } from "./filewriter";
import { LoggingBase } from "@hibas123/logging"; import { LoggingBase } from "@hibas123/logging";
import Logging from "@hibas123/logging"; import Logging from "@hibas123/logging";
@ -21,8 +21,8 @@ LoggingBase.nativeFunctions = {
}, },
}; };
export const DefaultFileAdapter = new FileAdapter("./logs/all.log"); export const DefaultLoggingFile = new LoggingFiles("./logs/all.log");
Logging.addAdapter(DefaultFileAdapter); Logging.addAdapter(DefaultLoggingFile);
export default Logging; export default Logging;

39
src/test.ts
View File

@ -1,7 +1,7 @@
import { randomBytes } from "crypto"; import { randomBytes } from "crypto";
import * as fs from "fs"; import * as fs from "fs";
import { LoggingBase } from "@hibas123/logging"; import { LoggingBase } from "@hibas123/logging";
import Logging, { DefaultFileAdapter, FileAdapter } from "."; import Logging, { DefaultLoggingFile } from ".";
const deleteFolderRecursive = function (path: string) { const deleteFolderRecursive = function (path: string) {
if (fs.existsSync(path)) { if (fs.existsSync(path)) {
@ -52,32 +52,19 @@ cus2.log("Hello from custom Logger 2");
cus22.log("Hello from custom Logger 22"); cus22.log("Hello from custom Logger 22");
cus2.log("Hello from custom Logger 2"); cus2.log("Hello from custom Logger 2");
const BenchmarkLogger = new LoggingBase({
console: false,
name: "bench",
});
BenchmarkLogger.addAdapter(DefaultLoggingFile);
async function benchmark(count: number, message_size: number) { async function benchmark(count: number, message_size: number) {
const BenchmarkLogger = new LoggingBase({ const randData = randomBytes(message_size).toString("hex");
console: false,
name: "bench",
});
if (fs.existsSync("logs/benchmark")) {
fs.unlinkSync("logs/benchmark");
}
if (fs.existsSync("logs/benchmark.old")) {
fs.unlinkSync("logs/benchmark.old");
}
const BenchmarkFile = new FileAdapter("logs/benchmark");
BenchmarkLogger.addAdapter(BenchmarkFile);
const randData = randomBytes(message_size / 2).toString("hex");
const t = process.hrtime(); const t = process.hrtime();
for (let i = 0; i < count; i++) { for (let i = 0; i < count; i++) {
BenchmarkLogger.log(randData); BenchmarkLogger.log(randData);
} }
await BenchmarkFile.flush(false);
await BenchmarkLogger.close();
const diff = process.hrtime(t); const diff = process.hrtime(t);
const NS_PER_SEC = 1e9; const NS_PER_SEC = 1e9;
const ns = diff[0] * NS_PER_SEC + diff[1]; const ns = diff[0] * NS_PER_SEC + diff[1];
@ -92,12 +79,12 @@ async function benchmark(count: number, message_size: number) {
const benchTimer = Logging.time("benchmark"); const benchTimer = Logging.time("benchmark");
Promise.resolve().then(async () => { Promise.resolve().then(async () => {
console.log("Large data benchmark:"); console.log("Large data benchmark:");
await benchmark(70000, 50000); await benchmark(7000, 50000);
console.log("Realdata data benchmark:"); console.log("Realdata data benchmark:");
await benchmark(100000, 100); await benchmark(100000, 100);
benchTimer.end(); benchTimer.end();
const timer = Logging.time("Test Timer");
setTimeout(() => timer.end(), 1000);
}); });
const timer = Logging.time("Test Timer");
setTimeout(() => timer.end(), 1000);