OpenServer/RealtimeDB-OLD
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Updating dependencies
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
User user 2021-06-02 10:11:52 +02:00
parent 6e561bd30f
commit 97a6b45c92
11 changed files with 6911 additions and 2577 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7490
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

45
package.json
View File

@ -14,33 +14,32 @@
"author": "Fabian Stamm <dev@fabianstamm.de>", "author": "Fabian Stamm <dev@fabianstamm.de>",
"license": "ISC", "license": "ISC",
"devDependencies": { "devDependencies": {
"@types/dotenv": "^8.2.0", "@types/jsonwebtoken": "^8.5.1",
"@types/jsonwebtoken": "^8.5.0", "@types/koa": "^2.13.3",
"@types/koa": "^2.11.6", "@types/koa-router": "^7.4.2",
"@types/koa-router": "^7.4.1",
"@types/leveldown": "^4.0.2", "@types/leveldown": "^4.0.2",
"@types/levelup": "^4.3.0", "@types/levelup": "^4.3.1",
"@types/nanoid": "^2.1.0", "@types/msgpack5": "^3.4.1",
"@types/node": "^14.14.5", "@types/node": "^14.17.2",
"@types/ws": "^7.2.8", "@types/ws": "^7.4.4",
"concurrently": "^5.3.0", "nodemon": "^2.0.7",
"nodemon": "^2.0.6", "ts-node": "^10.0.0",
"ts-node": "^9.0.0", "typescript": "^4.3.2"
"typescript": "^4.0.5"
}, },
"dependencies": { "dependencies": {
"@hibas123/nodelogging": "^2.4.5", "@hibas123/logging": "^3.1.2",
"@hibas123/utils": "^2.2.16", "@hibas123/nodelogging": "^3.1.3",
"dotenv": "^8.2.0", "@hibas123/utils": "^2.2.18",
"handlebars": "^4.7.6", "dotenv": "^10.0.0",
"handlebars": "^4.7.7",
"jsonwebtoken": "^8.5.1", "jsonwebtoken": "^8.5.1",
"koa": "^2.13.0", "koa": "^2.13.1",
"koa-body": "^4.2.0", "koa-body": "^4.2.0",
"koa-router": "^9.4.0", "koa-router": "^10.0.0",
"leveldown": "^5.6.0", "leveldown": "^6.0.0",
"levelup": "^4.4.0", "levelup": "^5.0.0",
"nanoid": "^3.1.16", "msgpack5": "^5.3.2",
"what-the-pack": "^2.0.3", "nanoid": "^3.1.23",
"ws": "^7.3.1" "ws": "^7.4.6"
} }
} }

1771
pnpm-lock.yaml Normal file
View File

File diff suppressed because it is too large Load Diff

2
src/database/lock.ts
View File

@ -12,7 +12,7 @@ export default class DocumentLock {
let key = collection + "/" + document; let key = collection + "/" + document;
let l = this.locks.get(key); let l = this.locks.get(key);
if (l) if (l)
await new Promise((resolve) => { await new Promise<void>((resolve) => {
l.push(resolve); l.push(resolve);
this.locks.set(key, l); this.locks.set(key, l);
}); });

10
src/database/query.ts
View File

@ -2,7 +2,7 @@ import { Database, Change, ChangeTypes } from "./database";
import { resNull } from "../storage"; import { resNull } from "../storage";
import * as nanoid from "nanoid"; import * as nanoid from "nanoid";
import Logging from "@hibas123/nodelogging"; import Logging from "@hibas123/nodelogging";
import * as MSGPack from "what-the-pack"; import * as MSGPack from "msgpack5";
import Session from "./session"; import Session from "./session";
import { LevelUpChain } from "levelup"; import { LevelUpChain } from "levelup";
import { Operations } from "../rules/parser"; import { Operations } from "../rules/parser";
@ -27,7 +27,9 @@ export type IQuery = ITypedQuery<
ICollectionQueries | IDocumentQueries | "snapshot" ICollectionQueries | IDocumentQueries | "snapshot"
>; >;
export const MP = MSGPack.initialize(2 ** 20); export const MP = MSGPack({});
// MSGPack.initialize(2 ** 20);
const ALPHABET = const ALPHABET =
"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
@ -110,7 +112,7 @@ export abstract class Query {
protected getDoc(collection: string, document: string) { protected getDoc(collection: string, document: string) {
return this.database.data return this.database.data
.get(Database.getKey(collection, document), { asBuffer: true }) .get(Database.getKey(collection, document), { asBuffer: true })
.then((res) => decode<any>(res as Buffer)) .then((res) => decode(res as Buffer))
.catch(resNull); .catch(resNull);
} }
@ -376,7 +378,7 @@ export class DocumentQuery extends Query {
} else { } else {
await this.database.data.put( await this.database.data.put(
Database.getKey(collection, document), Database.getKey(collection, document),
encode(data) encode(data).slice(0)
); );
} }

60
src/storage.ts
View File

@ -4,49 +4,53 @@ if (!fs.existsSync("./databases/")) {
fs.mkdirSync("./databases"); fs.mkdirSync("./databases");
} }
import LevelUp, { LevelUp as LU } from "levelup"; import * as LUR from "levelup";
import LevelDown, { LevelDown as LD } from "leveldown"; import * as LDR from "leveldown";
const LevelUp = LUR as any;
const LevelDown = LDR as any;
import type { LevelUp as LU } from "levelup";
import type { LevelDown as LD } from "leveldown";
import { AbstractIterator } from "abstract-leveldown"; import { AbstractIterator } from "abstract-leveldown";
export type LevelDB = LU<LD, AbstractIterator<any, any>>; export type LevelDB = LU<LD, AbstractIterator<any, any>>;
export type DBSet = { data: LevelDB, collection: LevelDB }; export type DBSet = { data: LevelDB; collection: LevelDB };
const databases = new Map<string, DBSet>(); const databases = new Map<string, DBSet>();
export function resNull(err): null { export function resNull(err): null {
if (!err.notFound) if (!err.notFound) throw err;
throw err;
return null; return null;
} }
async function rmRecursice(path: string) { async function rmRecursice(path: string) {
if (fs.existsSync(path)) { if (fs.existsSync(path)) {
await Promise.all(fs.readdirSync(path).map(async (file) => { await Promise.all(
var curPath = path + "/" + file; fs.readdirSync(path).map(async (file) => {
if (fs.lstatSync(curPath).isDirectory()) { // recurse var curPath = path + "/" + file;
await rmRecursice(curPath); if (fs.lstatSync(curPath).isDirectory()) {
} else { // delete file // recurse
await fs.promises.unlink(curPath); await rmRecursice(curPath);
} } else {
})); // delete file
await fs.promises.unlink(curPath);
}
})
);
await fs.promises.rmdir(path); await fs.promises.rmdir(path);
} }
}; }
export async function deleteLevelDB(name: string) { export async function deleteLevelDB(name: string) {
if (!name || name === "") if (!name || name === "") return;
return;
let db = databases.get(name); let db = databases.get(name);
if (db) { if (db) {
if (db.data.isOpen()) if (db.data.isOpen()) await db.data.close();
await db.data.close() if (db.collection.isOpen()) await db.collection.close();
if (db.collection.isOpen())
await db.collection.close()
} }
//TODO make sure, that name doesn't make it possible to delete all databases :) //TODO make sure, that name doesn't make it possible to delete all databases :)
await rmRecursice("./databases/" + name); await rmRecursice("./databases/" + name);
} }
@ -60,9 +64,15 @@ export default function getLevelDB(name: string): DBSet {
} }
db = { db = {
data: db && db.data.isOpen() ? db.data : LevelUp(LevelDown("./databases/" + name + "/data")), data:
collection: db && db.collection.isOpen() ? db.collection : LevelUp(LevelDown("./databases/" + name + "/collection")) db && db.data.isOpen()
} ? db.data
: LevelUp(LevelDown("./databases/" + name + "/data")),
collection:
db && db.collection.isOpen()
? db.collection
: LevelUp(LevelDown("./databases/" + name + "/collection")),
};
databases.set(name, db); databases.set(name, db);
return db; return db;

49
src/web/helper/error.ts
View File

@ -7,29 +7,30 @@ export default function RequestError(ctx: Context, next) {
ctx.body = message; ctx.body = message;
} }
return next().then(() => { return next()
if (ctx.status === HttpStatusCode.NOT_FOUND) { .then(() => {
reply(HttpStatusCode.NOT_FOUND, "Not found"); if (ctx.status === HttpStatusCode.NOT_FOUND) {
} reply(HttpStatusCode.NOT_FOUND, "Not found");
}).catch(error => { }
let message = "Internal server error"; })
let status = HttpStatusCode.INTERNAL_SERVER_ERROR; .catch((error) => {
if (typeof error === "string") { let message = "Internal server error";
message = error; let status = HttpStatusCode.INTERNAL_SERVER_ERROR;
} else if (!(error instanceof HttpError)) { if (typeof error === "string") {
Logging.error(error); message = error;
message = error.message; } else if (!(error instanceof HttpError)) {
} else {
if (error.status === HttpStatusCode.INTERNAL_SERVER_ERROR) {
//If internal server error log whole error
Logging.error(error); Logging.error(error);
message = error.message;
} else {
if (error.status === HttpStatusCode.INTERNAL_SERVER_ERROR) {
//If internal server error log whole error
Logging.error(error);
} else {
message = error.message.split("\n", 1)[0];
Logging.error(message);
}
status = error.status;
} }
else { reply(status, message);
message = error.message.split("\n", 1)[0]; });
Logging.errorMessage(message); }
}
status = error.status;
}
reply(status, message);
})
};

45
src/web/helper/log.ts
View File

@ -1,27 +1,40 @@
import { LoggingBase } from "@hibas123/nodelogging"; import { LoggingBase } from "@hibas123/logging";
import { FileAdapter } from "@hibas123/nodelogging";
import { Context } from "koa"; import { Context } from "koa";
import config from "../../config"; import config from "../../config";
const route_logging = new LoggingBase({ name: "access", files: { errorfile: null }, console: config.dev }) const route_logging = new LoggingBase({
name: "access",
console: config.dev,
});
route_logging.addAdapter(new FileAdapter("logs/access.log"));
const RequestLog = async (ctx: Context, next) => { const RequestLog = async (ctx: Context, next) => {
if (!config.access_log) return next(); if (!config.access_log) return next();
let start = process.hrtime() let start = process.hrtime();
let to = false let to = false;
let print = () => { let print = () => {
let td = process.hrtime(start) let td = process.hrtime(start);
let time = !to ? (td[0] * 1e3 + td[1] / 1e6).toFixed(2) : "--.--" let time = !to ? (td[0] * 1e3 + td[1] / 1e6).toFixed(2) : "--.--";
let resColor = "" let resColor = "";
let status = ctx.status; let status = ctx.status;
if (status >= 200 && status < 300) resColor = "\x1b[32m" //Green if (status >= 200 && status < 300) resColor = "\x1b[32m";
else if (status === 304 || status === 302) resColor = "\x1b[33m" //Green
else if (status >= 400 && status < 500) resColor = "\x1b[36m" //Cyan else if (status === 304 || status === 302) resColor = "\x1b[33m";
else if (status >= 500 && status < 600) resColor = "\x1b[31m" //Red else if (status >= 400 && status < 500) resColor = "\x1b[36m";
let m = ctx.method //Cyan
while (m.length < 4) m += " " else if (status >= 500 && status < 600) resColor = "\x1b[31m"; //Red
let message = `${m} ${ctx.originalUrl.split("?", 1)[0]} ${resColor}${status}\x1b[0m - ${time}ms`; let m = ctx.method;
while (m.length < 4) m += " ";
let message = `${m} ${
ctx.originalUrl.split("?", 1)[0]
} ${resColor}${status}\x1b[0m - ${time}ms`;
route_logging.log(message); route_logging.log(message);
} };
let timeout = new Promise((yes) => setTimeout(() => (to = true) && yes(), 10000)); let timeout = new Promise<void>((yes) =>
setTimeout(() => (to = true) && yes(), 10000)
);
await Promise.race([timeout, next()]); await Promise.race([timeout, next()]);
print(); print();
}; };

8
src/web/v1/admin.ts
View File

@ -51,7 +51,7 @@ AdminRoute.get("/settings", async (ctx) => {
AdminRoute.get("/data", async (ctx) => { AdminRoute.get("/data", async (ctx) => {
const { database } = ctx.query; const { database } = ctx.query;
let db = DatabaseManager.getDatabase(database); let db = DatabaseManager.getDatabase(database as string);
if (!db) throw new BadRequestError("Database not found"); if (!db) throw new BadRequestError("Database not found");
let res = await new Promise<string[][]>((yes, no) => { let res = await new Promise<string[][]>((yes, no) => {
const stream = db.data.createReadStream({ const stream = db.data.createReadStream({
@ -130,7 +130,7 @@ AdminRoute.get("/database", (ctx) => {
AdminRoute.get("/collections", async (ctx) => { AdminRoute.get("/collections", async (ctx) => {
const { database } = ctx.query; const { database } = ctx.query;
let db = DatabaseManager.getDatabase(database); let db = DatabaseManager.getDatabase(database as string);
if (!db) throw new BadRequestError("Database not found"); if (!db) throw new BadRequestError("Database not found");
let res = await new Promise<string[]>((yes, no) => { let res = await new Promise<string[]>((yes, no) => {
@ -156,7 +156,7 @@ AdminRoute.get("/collections", async (ctx) => {
AdminRoute.get("/collections/cleanup", async (ctx) => { AdminRoute.get("/collections/cleanup", async (ctx) => {
const { database } = ctx.query; const { database } = ctx.query;
let db = DatabaseManager.getDatabase(database); let db = DatabaseManager.getDatabase(database as string);
if (!db) throw new BadRequestError("Database not found"); if (!db) throw new BadRequestError("Database not found");
let deleted = await db.runCleanup(); let deleted = await db.runCleanup();
@ -184,7 +184,7 @@ AdminRoute.get(
AdminRoute.get("/database/update", async (ctx) => { AdminRoute.get("/database/update", async (ctx) => {
const { database } = ctx.query; const { database } = ctx.query;
let db = DatabaseManager.getDatabase(database); let db = DatabaseManager.getDatabase(database as string);
if (!db) throw new NotFoundError("Database not found!"); if (!db) throw new NotFoundError("Database not found!");
getForm("/v1/admin/database", "Change Database", { getForm("/v1/admin/database", "Change Database", {
name: { name: {

2
src/web/v1/index.ts
View File

@ -37,7 +37,7 @@ V1.post("/db/:database/query", async (ctx) => {
} }
if (authkey && db.publickey) { if (authkey && db.publickey) {
let res = await verifyJWT(authkey, db.publickey); let res = await verifyJWT(authkey as string, db.publickey);
if (res && !res.uid && res.user) res.uid = res.user; if (res && !res.uid && res.user) res.uid = res.user;
if (!res || !res.uid) { if (!res || !res.uid) {
throw new BadRequestError("Invalid JWT"); throw new BadRequestError("Invalid JWT");

2
src/websocket.ts
View File

@ -147,7 +147,7 @@ export class WebsocketConnectionManager {
h(message.data); h(message.data);
} }
} catch (err) { } catch (err) {
Logging.errorMessage("Unknown Error:"); Logging.error("Unknown Error:");
Logging.error(err); Logging.error(err);
} }
}); });