Working toward web compatibility

- Separating File output from LoggingBasse
- Separating Console output from LoggingBase
- Adding new Plugin mechanism
This commit is contained in:
Fabian Stamm 2019-03-23 16:50:12 +01:00
parent dec35001e3
commit 58ff2fd2ea
9 changed files with 2994 additions and 305 deletions

2551
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -1,6 +1,6 @@
{ {
"name": "@hibas123/nodelogging", "name": "@hibas123/nodelogging",
"version": "1.4.0", "version": "1.5.0-alpha.1",
"description": "", "description": "",
"main": "out/index.js", "main": "out/index.js",
"types": "out/index.d.ts", "types": "out/index.d.ts",
@ -21,5 +21,8 @@
"@types/node": "^8.0.24", "@types/node": "^8.0.24",
"nodemon": "^1.17.4", "nodemon": "^1.17.4",
"typescript": "^2.4.2" "typescript": "^2.4.2"
},
"dependencies": {
"@hibas123/utils": "^2.0.0"
} }
} }

View File

@ -30,20 +30,66 @@ NodeLogging can work without any configuration, but it may be useful to change t
Todo so you are capable of creating own instances of the LoggingBase class Todo so you are capable of creating own instances of the LoggingBase class
``` javascript ``` javascript
const CustomLogging = new LoggingBase({ const CustomLogging = new LoggingBase(name | {
name: "custom", name: "custom", // default undefined
logfile: "./logs/test.log", files: true | false | { //default true
errorfile: "/var/log/custom.err", logfile: "./logs/test.log",
console_out: false errorfile: "/var/log/custom.err",
}
console: false // default true
}); });
``` ```
The name property prefixes the console output with the name. Also if no logfile or errorfile is created the following standard values are used: The name property prefixes the console output with the name. Also if no logfile or errorfile is created the following standard values are used:
./logs/all.{name}.log - ./logs/all.{name}.log
./logs/error.{name}.log - ./logs/error.{name}.log
To not use any logfiles just set files to false.
# Plugins
There is a new Plugin API available, that makes is possible to add custom Logging Adapter.
``` javascript
const Demo = new LoggingBase("Demo");
Demo.addAdapter(new DemoAdapter({ color: "rainbow" }));
```
The adapters need to provide a very simple Interface:
``` typescript
interface Adapter {
init(observable: ObservableInterface<Message>, name?: string): void | Promise<void>;
flush(sync: true): void;
flush(sync: false): void | Promise<void>;
}
interface Message {
type: LoggingTypes;
name?:string;
text: {
raw: string[],
formatted: string[]
};
date: Date;
file: string;
customColors?:string;
}
enum LoggingTypes {
Log,
Warning,
Error,
Debug
}
```
The `ObservableInterface` comes from `@hibas123/utils`. It provides a very simple api for subscribing and unsubscribing from the message events.
More Details on Observable [here](https://git.stamm.me/OpenServer/Utils)
To not use any logfiles just set the values to null.
# License # License
MIT MIT

39
src/consolewriter.ts Normal file
View File

@ -0,0 +1,39 @@
import {Adapter, Message, LoggingTypes} from "./types";
import { ObservableInterface } from "@hibas123/utils"
import { Colors } from "./index";
export class ConsoleWriter implements Adapter {
init(observable:ObservableInterface<Message>) {
observable.subscribe(this.onMessage.bind(this));
}
flush() {}
onMessage(message:Message) {
let consoleLogFormat = Colors.Reset;
if (!message.customColors) {
switch (message.type) {
case LoggingTypes.Log:
//m += FgWhite + BgBlack;
break;
case LoggingTypes.Error:
consoleLogFormat += Colors.FgRed;//FgWhite + BgRed + FgWhite;
break;
case LoggingTypes.Debug:
consoleLogFormat += Colors.FgCyan;
break;
case LoggingTypes.Warning:
consoleLogFormat += Colors.FgYellow;
break;
}
} else {
consoleLogFormat += message.customColors;
}
let lines = message.text.formatted;
let name = "";
if(message.name) name = `[${message.name}]=>`;
lines.forEach(line=>console.log(consoleLogFormat + name + line + Colors.Reset))
}
}

208
src/filewriter.ts Normal file
View File

@ -0,0 +1,208 @@
import { Lock, ObservableInterface } from "@hibas123/utils"
import * as path from "path";
import * as fs from "fs";
import { Adapter, Message, LoggingTypes } from "./types";
const maxFileSize = 500000000;
export class LoggingFiles implements Adapter {
file: Files;
constructor(filename: string, private error = false) {
this.file = Files.getFile(filename);
}
init(observable: ObservableInterface<Message>) {
observable.subscribe(this.onMessage.bind(this));
return this.file.init();
}
flush(sync: boolean) {
this.file.flush(sync);
}
onMessage(message: Message) {
// Just ignore all non error messages, if this.error is set
if (this.error && message.type !== LoggingTypes.Error)
return;
let txt = message.text.formatted.join("\n") + "\n";
txt = txt.replace(/[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g, "");
let index = txt.indexOf("\x1b");
while (index >= 0) {
txt = txt.substring(0, index) + txt.substring(index + 5, txt.length);
index = txt.indexOf("\x1b");
}
let msg = Buffer.from(txt);
this.file.write(msg);
}
}
export class Files {
private static files = new Map<string, Files>();
static getFile(filename: string): Files {
filename = path.resolve(filename);
let file = this.files.get(filename);
if (!file) {
file = new Files(filename);
this.files.set(filename, file);
}
return file;
}
private size: number = 0;
private stream: fs.WriteStream = undefined;
private lock = new Lock();
public initialized = false;
private constructor(private file: string) { }
public async init() {
if (this.initialized)
return;
let lock = await this.lock.getLock();
await this.initializeFile()
this.initialized = true;
lock.release();
this.checkQueue()
}
private async initializeFile(new_file = false) {
try {
if (this.stream) {
this.stream.close();
}
const folder = path.dirname(this.file);
if (folder) {
if (!await fsExists(folder)) {
await fsMkDir(folder).catch(() => { }); //Could happen, if two seperate instances want to create the same folder so ignoring
}
}
let size = 0;
if (await fsExists(this.file)) {
let stats = await fsStat(this.file);
if (new_file || stats.size >= maxFileSize) {
if (await fsExists(this.file + ".old"))
await fsUnlink(this.file + ".old");
await fsMove(this.file, this.file + ".old")
} else {
size = stats.size;
}
}
this.stream = fs.createWriteStream(this.file, { flags: "a" })
this.size = size;
} catch (e) {
console.log(e);
//ToDo is this the right behavior?
process.exit(1);
}
}
private queue: Buffer[] = [];
async checkQueue() {
if (this.lock.locked) return;
let lock = await this.lock.getLock();
let msg: Buffer;
while (msg = this.queue.shift()) {
await this.write_to_file(msg);
}
lock.release();
}
public flush(sync: boolean) {
if (sync) {
// if sync flush, the process most likely is in failstate, so checkQueue stopped its work.
let msg: Buffer;
while (msg = this.queue.shift()) {
this.stream.write(msg);
}
} else {
return Promise.resolve().then(async () => {
const lock = await this.lock.getLock();
lock.release();
await this.checkQueue();
})
}
}
private async write_to_file(data: Buffer) {
try {
if (data.byteLength < maxFileSize && this.size + data.byteLength > maxFileSize) {
await this.initializeFile(true)
}
this.size += data.byteLength;
this.stream.write(data);
} catch (err) {
console.error(err);
this.initializeFile(false);
this.write_to_file(data);
}
}
public write(data: Buffer) {
this.queue.push(data);
this.checkQueue()
}
}
function fsUnlink(path) {
return new Promise((resolve, reject) => {
fs.unlink(path, (err) => {
if (err) reject(err);
else resolve();
})
})
}
function fsStat(path: string) {
return new Promise<fs.Stats>((resolve, reject) => {
fs.stat(path, (err, stats) => {
if (err) reject(err);
else resolve(stats);
})
})
}
function fsMove(oldPath: string, newPath: string) {
return new Promise((resolve, reject) => {
let callback = (err?) => {
if (err) reject(err)
else resolve()
}
fs.rename(oldPath, newPath, function (err) {
if (err) {
if (err.code === 'EXDEV') {
copy();
} else {
callback(err)
}
return;
}
callback()
});
function copy() {
fs.copyFile(oldPath, newPath, (err) => {
if (err) callback(err)
else fs.unlink(oldPath, callback);
})
}
})
}
function fsExists(path: string) {
return new Promise<boolean>((resolve, reject) => {
fs.exists(path, resolve);
});
}
function fsMkDir(path: string) {
return new Promise((resolve, reject) => {
fs.mkdir(path, (err) => err ? reject(err) : resolve());
});
}

View File

@ -2,7 +2,11 @@ import * as util from "util";
import * as fs from "fs"; import * as fs from "fs";
import { EventEmitter } from "events"; import { EventEmitter } from "events";
import * as path from "path"; import * as path from "path";
import Lock from "./lock"; import { Lock, Observable } from "@hibas123/utils";
import { Adapter, LoggingTypes, Message } from "./types";
import { ConsoleWriter } from "./consolewriter";
const OriginalErrorStackFunction = (<any>Error.prototype).prepareStackTrace
export const Colors = { export const Colors = {
Reset: "\x1b[0m", Reset: "\x1b[0m",
@ -32,131 +36,36 @@ export const Colors = {
BgWhite: "\x1b[47m" BgWhite: "\x1b[47m"
} }
const maxFileSize = 500000000;
const OriginalErrorStackFunction = (<any>Error.prototype).prepareStackTrace
export interface LoggingBaseOptions { export interface LoggingBaseOptions {
/** /**
* Name will be prefixed on Console output and added to logfiles, if not specified here * Name will be prefixed on Console output and added to logfiles, if not specified here
*/ */
name: string, name: string,
/** files: boolean | {
* Filename/path of the logfile. Skip if generated with name /**
*/ * Filename/path of the logfile. Skip if generated with name
logfile: string; */
/** logfile: string;
* Filename/path of the logfile. Skip if generated with name /**
*/ * Filename/path of the logfile. Skip if generated with name
errorfile: string; */
errorfile: string;
}
/** /**
* Prints output to console * Prints output to console
*/ */
console_out: boolean; console: boolean;
} }
class LoggingFiles {
private static files: LoggingFiles[] = [];
static getFile(filename: string): LoggingFiles {
filename = path.resolve(filename);
let file = this.files.find(e => e.file === filename);
if (!file) {
file = new LoggingFiles(filename);
this.files.push(file);
}
return file;
}
private file: string;
private size: number = 0;
private stream: fs.WriteStream = undefined;
private lock = new Lock();
private constructor(file: string) {
this.file = path.resolve(file);
this.init();
}
public async awaitinit() {
(await this.lock.getLock()).release();
}
private async init() {
let lock = await this.lock.getLock();
await this.initializeFile()
lock.release();
this.checkQueue()
}
private async initializeFile(new_file = false) {
try {
if (this.stream) {
this.stream.close();
}
const folder = path.dirname(this.file);
if (folder) {
if (!await fsExists(folder)) {
await fsMkDir(folder).catch(() => { }); //Could happen, if two seperate instances want to create the same folder so ignoring
}
}
let size = 0;
if (await fsExists(this.file)) {
let stats = await fsStat(this.file);
if (new_file || stats.size >= maxFileSize) {
if (await fsExists(this.file + ".old"))
await fsUnlink(this.file + ".old");
await fsMove(this.file, this.file + ".old")
} else {
size = stats.size;
}
}
this.stream = fs.createWriteStream(this.file, { flags: "a" })
this.size = size;
} catch (e) {
console.log(e);
//ToDo is this the right behavior?
process.exit(1);
}
}
private queue: Buffer[] = [];
async checkQueue() {
if (this.lock.locked) return;
let lock = await this.lock.getLock();
let msg: Buffer;
while (msg = this.queue.shift()) {
await this.write_to_file(msg);
}
lock.release();
}
private async write_to_file(data: Buffer) {
try {
if (data.byteLength < maxFileSize && this.size + data.byteLength > maxFileSize) {
await this.initializeFile(true)
}
this.size += data.byteLength;
this.stream.write(data);
} catch (err) {
console.error(err);
this.initializeFile(false);
this.write_to_file(data);
}
}
public write(data: Buffer) {
this.queue.push(data);
this.checkQueue()
}
}
export class LoggingBase { export class LoggingBase {
private config: LoggingBaseOptions; private logFile: any;
private errorFile: any;
private adapter: Adapter[] = [];
private adapter_init: Promise<void>[] = [];
private logFile: LoggingFiles; private messageObservable = new Observable<Message>();
private errorFile: LoggingFiles; private name: string;
constructor(options?: Partial<LoggingBaseOptions> | string) { constructor(options?: Partial<LoggingBaseOptions> | string) {
let opt: Partial<LoggingBaseOptions>; let opt: Partial<LoggingBaseOptions>;
@ -166,48 +75,69 @@ export class LoggingBase {
} else { } else {
opt = options; opt = options;
} }
if (opt.name) {
if (opt.logfile === undefined) {
opt.logfile = `./logs/all.${opt.name}.log`
}
if (opt.errorfile === undefined) { let config = {
opt.errorfile = `./logs/error.${opt.name}.log` name: undefined,
console: true,
files: true,
...opt
};
if (typeof config.files === "boolean") {
if (config.files === true) {
let name = "";
if (config.name)
name = "." + config.name;
config.files = {
logfile: `./logs/all${name}.log`,
errorfile: `./logs/error${name}.log`
}
} else {
config.files = undefined;
} }
} }
this.config = Object.assign(<LoggingBaseOptions>{
name: undefined, if (config.name)
console_out: true, this.name = config.name;
logfile: "./logs/all.log",
errorfile: "./logs/error.log"
}, opt);
for (let key in this) { for (let key in this) {
if (typeof this[key] === "function") this[key] = (<any>this[key]).bind(this); if (typeof this[key] === "function") this[key] = (<any>this[key]).bind(this);
} }
if (this.config.logfile) { if (typeof config.files !== "boolean" && config.files !== undefined && (config.files.logfile || config.files.errorfile)) {
this.logFile = LoggingFiles.getFile(this.config.logfile); const { LoggingFiles } = require("./filewriter");
if (config.files.logfile) {
this.addAdapter(new LoggingFiles(config.files.logfile));
}
if (config.files.errorfile) {
this.addAdapter(new LoggingFiles(config.files.errorfile, true));
}
} }
if (this.config.errorfile) { if (config.console) {
this.errorFile = LoggingFiles.getFile(this.config.errorfile); this.addAdapter(new ConsoleWriter());
} }
} }
get console_out() { addAdapter(adapter: Adapter) {
return this.config.console_out; this.adapter.push(adapter);
let prms = Promise.resolve(adapter.init(this.messageObservable.getPublicApi(), this.name));
this.adapter_init.push(prms);
} }
set console_out(value: boolean) { flush(sync: true): void;
this.config.console_out = value; flush(sync: false): Promise<void>;
flush(sync: boolean): void | Promise<void> {
if (sync) {
this.adapter.forEach(elm => elm.flush(true));
} else {
return Promise.all(this.adapter.map(elm => elm.flush(false))).then(() => { });
}
} }
public waitForSetup() { public waitForSetup() {
let w = []; return Promise.all(this.adapter_init);
if (this.logFile) w.push(this.logFile.awaitinit());
if (this.errorFile) w.push(this.errorFile.awaitinit());
return Promise.all(w)
} }
public events: EventEmitter = new EventEmitter(); public events: EventEmitter = new EventEmitter();
@ -243,26 +173,10 @@ export class LoggingBase {
} }
private message(type: LoggingTypes, message: any[] | string, customColors?: string, caller?: { file: string, line: number }) { private message(type: LoggingTypes, message: any[] | string, customColors?: string, caller?: { file: string, line: number }) {
var consoleLogFormat = Colors.Reset; let file_raw = caller || getCallerFile();
if (!customColors) { let file = `${file_raw.file}:${String(file_raw.line).padEnd(3, " ")}`;
switch (type) {
case LoggingTypes.Log: let mb = "";
//m += FgWhite + BgBlack;
break;
case LoggingTypes.Error:
consoleLogFormat += Colors.FgRed;//FgWhite + BgRed + FgWhite;
break;
case LoggingTypes.Debug:
consoleLogFormat += Colors.FgCyan;
break;
case LoggingTypes.Warning:
consoleLogFormat += Colors.FgYellow;
break;
}
} else {
consoleLogFormat += customColors;
}
var mb = "";
if (typeof message === "string") { if (typeof message === "string") {
mb = message; mb = message;
} else { } else {
@ -275,35 +189,27 @@ export class LoggingBase {
} }
}); });
} }
let file = caller || getCallerFile();
let lines = mb.split("\n");
let date = new Date().toISOString().replace(/T/, ' ').replace(/\..+/, ''); let date = new Date().toISOString().replace(/T/, ' ').replace(/\..+/, '');
let prefix = `[ ${date} ][${LoggingTypes[type].toUpperCase()}][${file.file}:${String(file.line).padEnd(3, " ")}]: `; let prefix = `[ ${date} ][${LoggingTypes[type].toUpperCase().padEnd(5, " ")}][${file}]: `;
let message_lines = mb.split("\n").map(line => prefix + line);
if (this.config.console_out) { let formatted = lines.map(line => prefix + line);
let name = "";
if (this.config.name) name = `[${this.config.name}]=>`; let msg: Message = {
message_lines.forEach(line => console.log(consoleLogFormat + name + line + Colors.Reset)); date: new Date(),
file,
name: this.name,
text: {
raw: lines,
formatted
},
type,
customColors
} }
let m = message_lines.join("\n"); this.messageObservable.send(msg);
m = m.replace(/[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g, "");
let index = m.indexOf("\x1b");
while (index >= 0) {
m = m.substring(0, index) + m.substring(index + 5, m.length);
index = m.indexOf("\x1b");
}
let data = Buffer.from(m + "\n")
if (type === LoggingTypes.Error && this.errorFile) {
this.errorFile.write(data);
}
if (this.logFile) {
this.logFile.write(data);
}
this.events.emit("message", { type: type, message: m, module: this.config.name });
} }
} }
@ -313,74 +219,7 @@ if (process.env.LOGGING_NO_DEFAULT !== "true") {
} }
export default Logging; export default Logging;
function fsUnlink(path) {
return new Promise((resolve, reject) => {
fs.unlink(path, (err) => {
if (err) reject(err);
else resolve();
})
})
}
function fsStat(path: string) {
return new Promise<fs.Stats>((resolve, reject) => {
fs.stat(path, (err, stats) => {
if (err) reject(err);
else resolve(stats);
})
})
}
function fsMove(oldPath: string, newPath: string) {
return new Promise((resolve, reject) => {
let callback = (err?) => {
if (err) reject(err)
else resolve()
}
fs.rename(oldPath, newPath, function (err) {
if (err) {
if (err.code === 'EXDEV') {
copy();
} else {
callback(err)
}
return;
}
callback()
});
function copy() {
fs.copyFile(oldPath, newPath, (err) => {
if (err) callback(err)
else fs.unlink(oldPath, callback);
})
// var readStream = fs.createReadStream(oldPath);
// var writeStream = fs.createWriteStream(newPath);
// readStream.on('error', callback);
// writeStream.on('error', callback);
// readStream.on('close', function () {
// fs.unlink(oldPath, callback);
// });
// readStream.pipe(writeStream);
}
})
}
function fsExists(path: string) {
return new Promise<boolean>((resolve, reject) => {
fs.exists(path, resolve);
});
}
function fsMkDir(path: string) {
return new Promise((resolve, reject) => {
fs.mkdir(path, (err) => err ? reject(err) : resolve());
});
}
function getStack() { function getStack() {
// Save original Error.prepareStackTrace // Save original Error.prepareStackTrace
@ -442,11 +281,4 @@ function getCallerFromExisting(err: Error): { file: string, line: number } {
} }
} }
} }
}
export enum LoggingTypes {
Log,
Warning,
Error,
Debug
} }

View File

@ -1,36 +0,0 @@
export type Release = { release: () => void };
export default class Lock {
private _locked: boolean = false;
get locked() {
return this._locked;
}
private toCome: (() => void)[] = [];
constructor() {
this.release = this.release.bind(this);
}
async getLock(): Promise<Release> {
if (!this._locked) return { release: this.lock() };
else {
return new Promise<Release>((resolve) => {
this.toCome.push(() => {
resolve({ release: this.lock() });
})
})
}
}
private lock() {
this._locked = true;
return this.release;
}
private async release() {
if (this.toCome.length > 0) {
this.toCome.shift()();
} else {
this._locked = false;
}
}
}

View File

@ -1,5 +1,21 @@
import { Logging, LoggingBase } from "./index"; import { Logging, LoggingBase } from "./index";
import { randomBytes } from "crypto"; import { randomBytes } from "crypto";
import * as fs from "fs";
const deleteFolderRecursive = function (path: string) {
if (fs.existsSync(path)) {
fs.readdirSync(path).forEach(function (file, index) {
var curPath = path + "/" + file;
if (fs.lstatSync(curPath).isDirectory()) { // recurse
deleteFolderRecursive(curPath);
} else { // delete file
fs.unlinkSync(curPath);
}
});
fs.rmdirSync(path);
}
};
deleteFolderRecursive("./logs")
Logging.log("test") Logging.log("test")
Logging.log("i", "am", { a: "an" }, 1000); Logging.log("i", "am", { a: "an" }, 1000);
@ -30,17 +46,20 @@ cus2.log("Hello from custom Logger 2")
cus22.log("Hello from custom Logger 22") cus22.log("Hello from custom Logger 22")
cus2.log("Hello from custom Logger 2") cus2.log("Hello from custom Logger 2")
Logging.console_out = false; const BenchmarkLogger = new LoggingBase({
console: false,
name: "bench"
})
async function benchmark(count: number, message_size: number) { async function benchmark(count: number, message_size: number) {
await Logging.waitForSetup(); await BenchmarkLogger.waitForSetup();
const randData = randomBytes(message_size).toString("hex") const randData = randomBytes(message_size).toString("hex")
const t = process.hrtime(); const t = process.hrtime();
for (let i = 0; i < count; i++) { for (let i = 0; i < count; i++) {
Logging.log(randData) BenchmarkLogger.log(randData)
} }
const diff = process.hrtime(t); const diff = process.hrtime(t);
const NS_PER_SEC = 1e9; const NS_PER_SEC = 1e9;
await Logging.waitForSetup(); await BenchmarkLogger.waitForSetup();
const ns = diff[0] * NS_PER_SEC + diff[1]; const ns = diff[0] * NS_PER_SEC + diff[1];
console.log(`Benchmark took ${ns / 1000000}ms for ${count} messages with a size of ${message_size} characters`); console.log(`Benchmark took ${ns / 1000000}ms for ${count} messages with a size of ${message_size} characters`);
console.log(`This is equal to ${(ns / 1000000) / count} ms per message`) console.log(`This is equal to ${(ns / 1000000) / count} ms per message`)

27
src/types.ts Normal file
View File

@ -0,0 +1,27 @@
import { ObservableInterface }from "@hibas123/utils";
export enum LoggingTypes {
Log,
Warning,
Error,
Debug
}
export interface Message {
type: LoggingTypes;
name?:string;
text: {
raw: string[],
formatted: string[]
};
date: Date;
file: string;
customColors?:string;
}
export interface Adapter {
init(observable: ObservableInterface<Message>, name?: string): void | Promise<void>;
flush(sync: true): void;
flush(sync: false): void | Promise<void>;
}