OpenServer/RealtimeDB-OLD
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Updating dependencies
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Fabian Stamm 2020-08-14 11:55:43 +02:00
parent 794820e1d3
commit 5cbc445597
6 changed files with 843 additions and 644 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1354
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

36
package.json
View File

@ -17,31 +17,31 @@
"license": "ISC", "license": "ISC",
"devDependencies": { "devDependencies": {
"@types/dotenv": "^8.2.0", "@types/dotenv": "^8.2.0",
"@types/jsonwebtoken": "^8.3.8", "@types/jsonwebtoken": "^8.5.0",
"@types/koa": "^2.11.2", "@types/koa": "^2.11.3",
"@types/koa-router": "^7.4.0", "@types/koa-router": "^7.4.1",
"@types/leveldown": "^4.0.2", "@types/leveldown": "^4.0.2",
"@types/levelup": "^4.3.0", "@types/levelup": "^4.3.0",
"@types/nanoid": "^2.1.0", "@types/nanoid": "^2.1.0",
"@types/node": "^13.9.3", "@types/node": "^14.0.27",
"@types/ws": "^7.2.3", "@types/ws": "^7.2.6",
"concurrently": "^5.1.0", "concurrently": "^5.3.0",
"nodemon": "^2.0.2", "nodemon": "^2.0.4",
"typescript": "^3.8.3" "typescript": "^3.9.7"
}, },
"dependencies": { "dependencies": {
"@hibas123/nodelogging": "^2.1.5", "@hibas123/nodelogging": "^2.4.5",
"@hibas123/utils": "^2.2.3", "@hibas123/utils": "^2.2.4",
"dotenv": "^8.2.0", "dotenv": "^8.2.0",
"handlebars": "^4.7.3", "handlebars": "^4.7.6",
"jsonwebtoken": "^8.5.1", "jsonwebtoken": "^8.5.1",
"koa": "^2.11.0", "koa": "^2.13.0",
"koa-body": "^4.1.1", "koa-body": "^4.2.0",
"koa-router": "^8.0.8", "koa-router": "^9.4.0",
"leveldown": "^5.5.1", "leveldown": "^5.6.0",
"levelup": "^4.3.2", "levelup": "^4.4.0",
"nanoid": "^2.1.11", "nanoid": "^3.1.12",
"what-the-pack": "^2.0.3", "what-the-pack": "^2.0.3",
"ws": "^7.2.3" "ws": "^7.3.1"
} }
} }

9
src/database/database.ts
View File

@ -12,12 +12,15 @@ import {
} from "./query"; } from "./query";
import Logging from "@hibas123/nodelogging"; import Logging from "@hibas123/nodelogging";
import Session from "./session"; import Session from "./session";
import nanoid = require("nanoid/generate"); import nanoid = require("nanoid");
import { Observable } from "@hibas123/utils"; import { Observable } from "@hibas123/utils";
const ALPHABET = const ALPHABET =
"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
const longNanoId = nanoid.customAlphabet(ALPHABET, 32);
const shortNanoId = nanoid.customAlphabet(ALPHABET, 16);
// interface ITransaction { // interface ITransaction {
// queries: ITypedQuery<IWriteQueries>[]; // queries: ITypedQuery<IWriteQueries>[];
// } // }
@ -156,7 +159,7 @@ export class Database {
.then((r) => r.toString()) .then((r) => r.toString())
.catch(resNull); .catch(resNull);
if (!collectionID && create) { if (!collectionID && create) {
collectionID = nanoid(ALPHABET, 32); collectionID = longNanoId();
await this.collections.put(key, collectionID); await this.collections.put(key, collectionID);
setImmediate(() => { setImmediate(() => {
this.collectionChangeListener.send({ this.collectionChangeListener.send({
@ -329,7 +332,7 @@ export class Database {
const { unsubscribe, value } = await query.snapshot(onchange); const { unsubscribe, value } = await query.snapshot(onchange);
const id = nanoid(ALPHABET, 16); const id = shortNanoId();
session.subscriptions.set(id, unsubscribe); session.subscriptions.set(id, unsubscribe);
return { return {
id, id,

66
src/database/query.ts
View File

@ -1,6 +1,6 @@
import { Database, Change, ChangeTypes } from "./database"; import { Database, Change, ChangeTypes } from "./database";
import { resNull } from "../storage"; import { resNull } from "../storage";
import nanoid = require("nanoid/generate"); import * as nanoid from "nanoid";
import Logging from "@hibas123/nodelogging"; import Logging from "@hibas123/nodelogging";
import * as MSGPack from "what-the-pack"; import * as MSGPack from "what-the-pack";
import Session from "./session"; import Session from "./session";
@ -31,6 +31,8 @@ export const MP = MSGPack.initialize(2 ** 20);
const ALPHABET = const ALPHABET =
"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
const longNanoID = nanoid.customAlphabet(ALPHABET, 32);
const { encode, decode } = MP; const { encode, decode } = MP;
type Runner = ( type Runner = (
@ -61,7 +63,7 @@ export abstract class Query {
*/ */
private validatePath(path: string[]) { private validatePath(path: string[]) {
return path.every( return path.every(
e => (e.match(/[^a-zA-Z0-9_\-\<\>]/g) || []).length === 0 (e) => (e.match(/[^a-zA-Z0-9_\-\<\>]/g) || []).length === 0
); );
} }
@ -106,7 +108,7 @@ export abstract class Query {
protected getDoc(collection: string, document: string) { protected getDoc(collection: string, document: string) {
return this.database.data return this.database.data
.get(Database.getKey(collection, document), { asBuffer: true }) .get(Database.getKey(collection, document), { asBuffer: true })
.then(res => decode<any>(res as Buffer)) .then((res) => decode<any>(res as Buffer))
.catch(resNull); .catch(resNull);
} }
@ -121,7 +123,7 @@ export abstract class Query {
document, document,
collection, collection,
data, data,
sender: this.session.id sender: this.session.id,
}; };
this.changes.push(change); this.changes.push(change);
@ -179,23 +181,25 @@ export abstract class Query {
const receivedChanges = (changes: Change[]) => { const receivedChanges = (changes: Change[]) => {
let res = changes let res = changes
.filter(change => this.checkChange(change)) .filter((change) => this.checkChange(change))
.map(change => { .map((change) => {
return { return {
id: change.document, id: change.document,
data: change.data, data: change.data,
type: change.type type: change.type,
}; };
}); });
if (res.length > 0) onChange(res); if (res.length > 0) onChange(res);
}; };
const unsub = this.database.collectionChangeListener.subscribe(change => { const unsub = this.database.collectionChangeListener.subscribe(
(change) => {
if (change.key === collectionKey) { if (change.key === collectionKey) {
if (change.type === "create") addSubscriber(change.id); if (change.type === "create") addSubscriber(change.id);
else removeSubscriber(); // Send delete for all elements (Don't know how to do this...) else removeSubscriber(); // Send delete for all elements (Don't know how to do this...)
} }
}); }
);
let { collection, document, collectionKey } = await this.database.resolve( let { collection, document, collectionKey } = await this.database.resolve(
this.query.path this.query.path
@ -236,7 +240,7 @@ export abstract class Query {
unsub(); unsub();
removeSubscriber(); removeSubscriber();
}, },
value: await this.firstSend(collection, document) value: await this.firstSend(collection, document),
}; };
} }
} }
@ -257,7 +261,7 @@ export class DocumentQuery extends Query {
createCollection: false, createCollection: false,
needDocument: false, needDocument: false,
permission: "read", permission: "read",
runner: this.get runner: this.get,
}; };
case "set": case "set":
return { return {
@ -265,7 +269,7 @@ export class DocumentQuery extends Query {
createCollection: true, createCollection: true,
needDocument: true, needDocument: true,
permission: "write", permission: "write",
runner: this.set runner: this.set,
}; };
case "update": case "update":
return { return {
@ -273,7 +277,7 @@ export class DocumentQuery extends Query {
createCollection: true, createCollection: true,
needDocument: true, needDocument: true,
permission: "write", permission: "write",
runner: this.update runner: this.update,
}; };
case "delete": case "delete":
return { return {
@ -281,7 +285,7 @@ export class DocumentQuery extends Query {
createCollection: false, createCollection: false,
needDocument: true, needDocument: true,
permission: "write", permission: "write",
runner: this.delete runner: this.delete,
}; };
default: default:
throw new Error("Invalid query type: " + type); throw new Error("Invalid query type: " + type);
@ -431,14 +435,14 @@ export class CollectionQuery extends Query {
prepare(query): IPreparedQuery { prepare(query): IPreparedQuery {
switch (query.type as ICollectionQueries) { switch (query.type as ICollectionQueries) {
case "add": case "add":
this._addId = nanoid(ALPHABET, 32); this._addId = longNanoID();
return { return {
batchCompatible: true, batchCompatible: true,
createCollection: true, createCollection: true,
needDocument: false, needDocument: false,
runner: this.add, runner: this.add,
permission: "write", permission: "write",
additionalLock: [...query.path, this._addId] additionalLock: [...query.path, this._addId],
}; };
case "get": case "get":
const limit = (query.options || {}).limit; const limit = (query.options || {}).limit;
@ -451,7 +455,7 @@ export class CollectionQuery extends Query {
createCollection: false, createCollection: false,
needDocument: false, needDocument: false,
permission: "read", permission: "read",
runner: this.get runner: this.get,
}; };
case "keys": case "keys":
return { return {
@ -459,7 +463,7 @@ export class CollectionQuery extends Query {
createCollection: false, createCollection: false,
needDocument: false, needDocument: false,
permission: "read", permission: "read",
runner: this.keys runner: this.keys,
}; };
case "list": case "list":
return { return {
@ -467,7 +471,7 @@ export class CollectionQuery extends Query {
createCollection: false, createCollection: false,
needDocument: false, needDocument: false,
permission: "read", permission: "read",
runner: this.keys runner: this.keys,
}; };
case "delete-collection": case "delete-collection":
return { return {
@ -475,7 +479,7 @@ export class CollectionQuery extends Query {
createCollection: false, createCollection: false,
needDocument: false, needDocument: false,
permission: "write", permission: "write",
runner: this.deleteCollection runner: this.deleteCollection,
}; };
// run = () => q.deleteCollection(); // run = () => q.deleteCollection();
// break; // break;
@ -489,7 +493,7 @@ export class CollectionQuery extends Query {
const invalidWhere = new QueryError("Invalid Where"); const invalidWhere = new QueryError("Invalid Where");
if (!Array.isArray(value)) throw invalidWhere; if (!Array.isArray(value)) throw invalidWhere;
let c = []; let c = [];
this._where = value.map(cond => { this._where = value.map((cond) => {
Logging.debug("Query Condition", cond); Logging.debug("Query Condition", cond);
if (Array.isArray(cond)) { if (Array.isArray(cond)) {
if (cond.length !== 3) throw invalidWhere; if (cond.length !== 3) throw invalidWhere;
@ -522,7 +526,7 @@ export class CollectionQuery extends Query {
type: "set", type: "set",
path: this.additionalLock, path: this.additionalLock,
data: this.query.data, data: this.query.data,
options: this.query.options options: this.query.options,
}); });
await q.run(collection, this._addId, batch, collectionKey); await q.run(collection, this._addId, batch, collectionKey);
return this._addId; return this._addId;
@ -538,7 +542,7 @@ export class CollectionQuery extends Query {
return { return {
gt, gt,
lt lt,
}; };
} }
@ -549,7 +553,7 @@ export class CollectionQuery extends Query {
let keys = []; let keys = [];
const stream = this.database.data.createKeyStream({ const stream = this.database.data.createKeyStream({
...this.getStreamOptions(collection), ...this.getStreamOptions(collection),
keyAsBuffer: false keyAsBuffer: false,
}); });
stream.on("data", (key: string) => { stream.on("data", (key: string) => {
let s = key.split("/", 2); let s = key.split("/", 2);
@ -589,7 +593,7 @@ export class CollectionQuery extends Query {
return val > value; return val > value;
case "array-contains": case "array-contains":
if (Array.isArray(val)) { if (Array.isArray(val)) {
return val.some(e => e === value); return val.some((e) => e === value);
} }
return false; return false;
@ -614,7 +618,7 @@ export class CollectionQuery extends Query {
const stream = this.database.data.iterator({ const stream = this.database.data.iterator({
...this.getStreamOptions(collection), ...this.getStreamOptions(collection),
keyAsBuffer: false, keyAsBuffer: false,
valueAsBuffer: true valueAsBuffer: true,
}); });
let values: DocRes[] = []; let values: DocRes[] = [];
@ -622,7 +626,7 @@ export class CollectionQuery extends Query {
const onValue = (err: Error, key: string, value: Buffer) => { const onValue = (err: Error, key: string, value: Buffer) => {
if (err) { if (err) {
no(err); no(err);
stream.end(err => Logging.error(err)); stream.end((err) => Logging.error(err));
} else { } else {
if (!key && !value) { if (!key && !value) {
// END // END
@ -639,10 +643,10 @@ export class CollectionQuery extends Query {
if (this.limit < 0 || values.length < this.limit) { if (this.limit < 0 || values.length < this.limit) {
values.push({ values.push({
id, id,
data data,
}); });
} else { } else {
stream.end(err => (err ? no(err) : yes(values))); stream.end((err) => (err ? no(err) : yes(values)));
return; return;
} }
} }
@ -670,7 +674,7 @@ export class CollectionQuery extends Query {
return new Promise<string[]>((yes, no) => { return new Promise<string[]>((yes, no) => {
let keys = []; let keys = [];
const stream = this.database.data.createKeyStream({ const stream = this.database.data.createKeyStream({
keyAsBuffer: false keyAsBuffer: false,
}); });
stream.on("data", (key: string) => keys.push(key.split("/"))); stream.on("data", (key: string) => keys.push(key.split("/")));
stream.on("end", () => yes(keys)); stream.on("end", () => yes(keys));
@ -701,7 +705,7 @@ export class CollectionQuery extends Query {
this.database.collectionChangeListener.send({ this.database.collectionChangeListener.send({
id: collection, id: collection,
key: collectionKey, key: collectionKey,
type: "delete" type: "delete",
}); });
} }
} finally { } finally {

8
src/web/v1/index.ts
View File

@ -4,18 +4,18 @@ import { DatabaseManager } from "../../database/database";
import { import {
NotFoundError, NotFoundError,
NoPermissionError, NoPermissionError,
BadRequestError BadRequestError,
} from "../helper/errors"; } from "../helper/errors";
import Logging from "@hibas123/nodelogging"; import Logging from "@hibas123/nodelogging";
import Session from "../../database/session"; import Session from "../../database/session";
import nanoid = require("nanoid"); import { nanoid } from "nanoid";
import { verifyJWT } from "../../helper/jwt"; import { verifyJWT } from "../../helper/jwt";
import { QueryError } from "../../database/query"; import { QueryError } from "../../database/query";
const V1 = new Router({ prefix: "/v1" }); const V1 = new Router({ prefix: "/v1" });
V1.use("/admin", AdminRoute.routes(), AdminRoute.allowedMethods()); V1.use("/admin", AdminRoute.routes(), AdminRoute.allowedMethods());
V1.post("/db/:database/query", async ctx => { V1.post("/db/:database/query", async (ctx) => {
const { database } = ctx.params; const { database } = ctx.params;
const { accesskey, authkey, rootkey } = ctx.query; const { accesskey, authkey, rootkey } = ctx.query;
@ -52,7 +52,7 @@ V1.post("/db/:database/query", async ctx => {
} }
} }
ctx.body = await db.run([query], session).catch(err => { ctx.body = await db.run([query], session).catch((err) => {
if (err instanceof QueryError) { if (err instanceof QueryError) {
throw new BadRequestError(err.message); throw new BadRequestError(err.message);
} }

2
src/websocket.ts
View File

@ -10,7 +10,7 @@ import {
} from "./database/query"; } from "./database/query";
import Session from "./database/session"; import Session from "./database/session";
import { verifyJWT } from "./helper/jwt"; import { verifyJWT } from "./helper/jwt";
import nanoid = require("nanoid"); import { nanoid } from "nanoid";
export class WebsocketConnectionManager { export class WebsocketConnectionManager {
static server: WebSocket.Server; static server: WebSocket.Server;