OpenServer/RealtimeDB-OLD
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Enabling rules
Some checks failed
continuous-integration/drone/tag Build is failing
Details

Browse Source
This commit is contained in:
Fabian Stamm
2020-03-24 15:16:21 +01:00
parent 88b0cb68d8
commit 1434036b42
6 changed files with 565 additions and 603 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.editorconfig
View File

@ -1,3 +1,5 @@
[*]
charset = utf-8
indent_size = 3
indent_style = space
indent_style = space
insert_final_newline = true

724
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

32
package.json
View File

@ -17,31 +17,31 @@
"license": "ISC",
"devDependencies": {
"@types/dotenv": "^8.2.0",
"@types/jsonwebtoken": "^8.3.5",
"@types/koa": "^2.11.0",
"@types/koa-router": "^7.0.42",
"@types/jsonwebtoken": "^8.3.8",
"@types/koa": "^2.11.2",
"@types/koa-router": "^7.4.0",
"@types/leveldown": "^4.0.2",
"@types/levelup": "^3.1.1",
"@types/levelup": "^4.3.0",
"@types/nanoid": "^2.1.0",
"@types/node": "^12.12.14",
"@types/ws": "^6.0.4",
"concurrently": "^5.0.0",
"nodemon": "^2.0.1",
"typescript": "^3.7.2"
"@types/node": "^13.9.3",
"@types/ws": "^7.2.3",
"concurrently": "^5.1.0",
"nodemon": "^2.0.2",
"typescript": "^3.8.3"
},
"dependencies": {
"@hibas123/nodelogging": "^2.1.2",
"@hibas123/nodelogging": "^2.1.5",
"@hibas123/utils": "^2.2.3",
"dotenv": "^8.2.0",
"handlebars": "^4.5.3",
"handlebars": "^4.7.3",
"jsonwebtoken": "^8.5.1",
"koa": "^2.11.0",
"koa-body": "^4.1.1",
"koa-router": "^7.4.0",
"leveldown": "^5.4.1",
"koa-router": "^8.0.8",
"leveldown": "^5.5.1",
"levelup": "^4.3.2",
"nanoid": "^2.1.7",
"nanoid": "^2.1.11",
"what-the-pack": "^2.0.3",
"ws": "^7.2.0"
"ws": "^7.2.3"
}
}
}

344
src/database/query.ts
View File

@ -7,7 +7,12 @@ import Session from "./session";
import { LevelUpChain } from "levelup";
export type IWriteQueries = "set" | "update" | "delete" | "add";
export type ICollectionQueries = "get" | "add" | "keys" | "delete-collection" | "list";
export type ICollectionQueries =
| "get"
| "add"
| "keys"
| "delete-collection"
| "list";
export type IDocumentQueries = "get" | "set" | "update" | "delete";
export interface ITypedQuery<T> {
@ -17,21 +22,30 @@ export interface ITypedQuery<T> {
options?: any;
}
export type IQuery = ITypedQuery<ICollectionQueries | IDocumentQueries | "snapshot">;
export type IQuery = ITypedQuery<
ICollectionQueries | IDocumentQueries | "snapshot"
>;
export const MP = MSGPack.initialize(2 ** 20);
const ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
const ALPHABET =
"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
const { encode, decode } = MP;
type Runner = (collection: string, document: string, batch: LevelUpChain, collectionKey: string) => any;
type Runner = (
collection: string,
document: string,
batch: LevelUpChain,
collectionKey: string
) => any;
interface IPreparedQuery {
createCollection: boolean;
needDocument: boolean;
batchCompatible: boolean;
runner: Runner;
permission: "write" | "read";
additionalLock?: string[];
}
@ -46,7 +60,9 @@ export abstract class Query {
* @param path Path to be checked
*/
private validatePath(path: string[]) {
return path.every(e => (e.match(/[^a-zA-Z0-9_\-\<\>]/g) || []).length === 0);
return path.every(
e => (e.match(/[^a-zA-Z0-9_\-\<\>]/g) || []).length === 0
);
}
public changes: Change[] = [];
@ -55,14 +71,24 @@ export abstract class Query {
public readonly needDocument: boolean;
public readonly batchCompatible: boolean;
public readonly additionalLock?: string[];
public readonly permission: string;
private readonly _runner: Runner;
constructor(protected database: Database, protected session: Session, protected query: IQuery, snapshot = false) {
constructor(
protected database: Database,
protected session: Session,
protected query: IQuery,
snapshot = false
) {
if (query.path.length > 10) {
throw new QueryError("Path is to long. Path is only allowed to be 10 Layers deep!");
throw new QueryError(
"Path is to long. Path is only allowed to be 10 Layers deep!"
);
}
if (!this.validatePath(query.path)) {
throw new QueryError("Path can only contain a-z A-Z 0-9 '-' '-' '<' and '>' ");
throw new QueryError(
"Path can only contain a-z A-Z 0-9 '-' '-' '<' and '>' "
);
}
if (!snapshot) {
@ -80,75 +106,113 @@ export abstract class Query {
protected getDoc(collection: string, document: string) {
return this.database.data
.get(Database.getKey(collection, document), { asBuffer: true })
.then(res => decode<any>(res as Buffer)).catch(resNull);
.then(res => decode<any>(res as Buffer))
.catch(resNull);
}
protected sendChange(collection: string, document: string, type: ChangeTypes, data: any) {
protected sendChange(
collection: string,
document: string,
type: ChangeTypes,
data: any
) {
let change: Change = {
type,
document,
collection,
data,
sender: this.session.id
}
};
this.changes.push(change);
}
protected static getConstructorParams(query: Query): [Database, Session, IQuery] {
protected static getConstructorParams(
query: Query
): [Database, Session, IQuery] {
return [query.database, query.session, query.query];
}
protected abstract checkChange(change: Change): boolean;
protected abstract firstSend(collection: string, document: string): Promise<any>;
protected abstract firstSend(
collection: string,
document: string
): Promise<any>;
public run(collection: string, document: string, batch: LevelUpChain, collectionKey: string) {
return this._runner.call(this, collection, document, batch, collectionKey);
public run(
collection: string,
document: string,
batch: LevelUpChain,
collectionKey: string
) {
let perm = this.database.rules.hasPermission(
this.query.path,
this.session
);
if (this.permission === "read" && !perm.read) {
throw new QueryError("No permission!");
} else if (this.permission === "write" && !perm.write) {
throw new QueryError("No permission!");
}
return this._runner.call(
this,
collection,
document,
batch,
collectionKey
);
}
public async snapshot(onChange: (change: (DocRes & { type: ChangeTypes })[]) => void) {
public async snapshot(
onChange: (change: (DocRes & { type: ChangeTypes })[]) => void
) {
let perm = this.database.rules.hasPermission(
this.query.path,
this.session
);
if (this.permission === "read" && !perm.read) {
throw new QueryError("No permission!");
}
const receivedChanges = (changes: Change[]) => {
let res = changes.filter(change => this.checkChange(change)).map(change => {
return {
id: change.document,
data: change.data,
type: change.type
}
})
if (res.length > 0)
onChange(res);
let res = changes
.filter(change => this.checkChange(change))
.map(change => {
return {
id: change.document,
data: change.data,
type: change.type
};
});
if (res.length > 0) onChange(res);
};
const unsub = this.database.collectionChangeListener.subscribe(change => {
if (change.key === collectionKey) {
if (change.type === "create")
addSubscriber(change.id);
else
removeSubscriber(); // Send delete for all elements (Don't know how to do this...)
if (change.type === "create") addSubscriber(change.id);
else removeSubscriber(); // Send delete for all elements (Don't know how to do this...)
}
})
});
let { collection, document, collectionKey } = await this.database.resolve(this.query.path)
let { collection, document, collectionKey } = await this.database.resolve(
this.query.path
);
let oldKey: string = undefined;
const removeSubscriber = () => {
if (!oldKey)
return;
if (!oldKey) return;
let s = this.database.changeListener.get(oldKey);
if (s) {
s.delete(receivedChanges);
if (s.size <= 0)
this.database.changeListener.delete(oldKey);
if (s.size <= 0) this.database.changeListener.delete(oldKey);
}
oldKey = undefined;
}
};
const addSubscriber = (collection: string) => {
const addSubscriber = () => {
let key = Database.getKey(collection, document);
if (oldKey !== key) {
if (oldKey !== undefined)
removeSubscriber();
if (oldKey !== undefined) removeSubscriber();
let s = this.database.changeListener.get(key);
if (!s) {
@ -158,10 +222,10 @@ export abstract class Query {
s.add(receivedChanges);
}
}
};
if (collection) {
addSubscriber(collection);
addSubscriber();
}
return {
@ -170,7 +234,7 @@ export abstract class Query {
removeSubscriber();
},
value: await this.firstSend(collection, document)
}
};
}
}
@ -178,7 +242,7 @@ interface UpdateData {
[path: string]: {
type: "value" | "timestamp" | "increment" | "push";
value: any;
}
};
}
export class DocumentQuery extends Query {
prepare(query: IQuery): IPreparedQuery {
@ -189,29 +253,33 @@ export class DocumentQuery extends Query {
batchCompatible: false,
createCollection: false,
needDocument: false,
permission: "read",
runner: this.get
}
};
case "set":
return {
batchCompatible: true,
createCollection: true,
needDocument: true,
permission: "write",
runner: this.set
}
};
case "update":
return {
batchCompatible: true,
createCollection: true,
needDocument: true,
permission: "write",
runner: this.update
}
};
case "delete":
return {
batchCompatible: true,
createCollection: false,
needDocument: true,
permission: "write",
runner: this.delete
}
};
default:
throw new Error("Invalid query type: " + type);
}
@ -225,22 +293,28 @@ export class DocumentQuery extends Query {
return this.getDoc(collection, document);
}
private async set(collection: string, document: string, batch?: LevelUpChain) {
private async set(
collection: string,
document: string,
batch?: LevelUpChain
) {
const { data, options } = this.query;
if (data === null)
return this.delete(collection, document, batch);
if (data === null) return this.delete(collection, document, batch);
let isNew = !(await this.getDoc(collection, document))
batch.put(Database.getKey(collection, document), encode(data))
this.sendChange(collection, document, isNew ? "added" : "modified", data)
let isNew = !(await this.getDoc(collection, document));
batch.put(Database.getKey(collection, document), encode(data));
this.sendChange(collection, document, isNew ? "added" : "modified", data);
}
private async update(collection: string, document: string, batch?: LevelUpChain) {
private async update(
collection: string,
document: string,
batch?: LevelUpChain
) {
const updateData: UpdateData = this.query.data;
let data = await this.getDoc(collection, document);
let isNew = false
let isNew = false;
if (!data) {
isNew = true;
data = {};
@ -252,8 +326,7 @@ export class DocumentQuery extends Query {
let parts = path.split(".");
while (parts.length > 1) {
let seg = parts.shift();
if (!data[seg])
data[seg] = {}
if (!data[seg]) data[seg] = {};
d = data[seg];
}
@ -290,23 +363,29 @@ export class DocumentQuery extends Query {
}
if (batch) {
batch.put(Database.getKey(collection, document), encode(data))
batch.put(Database.getKey(collection, document), encode(data));
} else {
await this.database.data
.put(Database.getKey(collection, document), encode(data))
await this.database.data.put(
Database.getKey(collection, document),
encode(data)
);
}
this.sendChange(collection, document, isNew ? "added" : "modified", data)
this.sendChange(collection, document, isNew ? "added" : "modified", data);
}
private async delete(collection: string, document: string, batch?: LevelUpChain) {
private async delete(
collection: string,
document: string,
batch?: LevelUpChain
) {
if (batch) {
batch.del(Database.getKey(collection, document))
batch.del(Database.getKey(collection, document));
} else {
await this.database.data.del(Database.getKey(collection, document));
}
this.sendChange(collection, document, "deleted", null)
this.sendChange(collection, document, "deleted", null);
}
checkChange(change: Change) {
@ -324,19 +403,19 @@ export class DocumentQuery extends Query {
type FieldPath = string;
type WhereFilterOp =
| '<'
| '<='
| '=='
| '>='
| '>'
| 'array-contains'
| 'in'
| 'array-contains-any';
| "<"
| "<="
| "=="
| ">="
| ">"
| "array-contains"
| "in"
| "array-contains-any";
interface IQueryWhereVerbose {
fieldPath: FieldPath,
opStr: WhereFilterOp,
value: any
fieldPath: FieldPath;
opStr: WhereFilterOp;
value: any;
}
type IQueryWhereArray = [FieldPath, WhereFilterOp, any];
@ -346,53 +425,55 @@ type IQueryWhere = IQueryWhereArray | IQueryWhereVerbose;
export class CollectionQuery extends Query {
private _addId: string;
prepare(query): IPreparedQuery {
switch (query.type as ICollectionQueries) {
case "add":
this._addId = nanoid(ALPHABET, 32)
this._addId = nanoid(ALPHABET, 32);
return {
batchCompatible: true,
createCollection: true,
needDocument: false,
runner: this.add,
permission: "write",
additionalLock: [...query.path, this._addId]
}
};
case "get":
const limit = (query.options || {}).limit;
if (limit)
this.limit = limit;
if (limit) this.limit = limit;
const where = (query.options || {}).where;
if (where)
this.where = where;
if (where) this.where = where;
return {
batchCompatible: false,
createCollection: false,
needDocument: false,
permission: "read",
runner: this.get
}
};
case "keys":
return {
batchCompatible: false,
createCollection: false,
needDocument: false,
permission: "read",
runner: this.keys
}
};
case "list":
return {
batchCompatible: false,
createCollection: false,
needDocument: false,
permission: "read",
runner: this.keys
}
};
case "delete-collection":
return {
batchCompatible: false,
createCollection: false,
needDocument: false,
permission: "write",
runner: this.deleteCollection
}
};
// run = () => q.deleteCollection();
// break;
default:
@ -400,32 +481,40 @@ export class CollectionQuery extends Query {
}
}
private _where: IQueryWhereArray[] = [];
public set where(value: IQueryWhere[]) {
const invalidWhere = new QueryError("Invalid Where");
if (!Array.isArray(value))
throw invalidWhere;
if (!Array.isArray(value)) throw invalidWhere;
let c = [];
this._where = value.map(cond => {
Logging.debug("Query Condition", cond);
if (Array.isArray(cond)) {
if (cond.length !== 3)
throw invalidWhere;
if (cond.length !== 3) throw invalidWhere;
return cond;
} else {
if (cond && typeof cond === "object" && "fieldPath" in cond && "opStr" in cond && "value" in cond) {
if (
cond &&
typeof cond === "object" &&
"fieldPath" in cond &&
"opStr" in cond &&
"value" in cond
) {
return [cond.fieldPath, cond.opStr, cond.value];
} else {
throw invalidWhere;
}
}
})
});
}
public limit: number = -1;
public async add(collection: string, document: string, batch: LevelUpChain, collectionKey: string) {
public async add(
collection: string,
document: string,
batch: LevelUpChain,
collectionKey: string
) {
let q = new DocumentQuery(this.database, this.session, {
type: "set",
path: this.additionalLock,
@ -442,28 +531,26 @@ export class CollectionQuery extends Query {
let lt = Buffer.alloc(gt.length);
lt.set(gt);
lt[gt.length - 1] = 0xFF;
lt[gt.length - 1] = 0xff;
return {
gt,
lt
}
};
}
public async keys(collection: string) {
if (!collection)
return []
if (!collection) return [];
return new Promise<string[]>((yes, no) => {
let keys = [];
const stream = this.database.data.createKeyStream({
...this.getStreamOptions(collection),
keyAsBuffer: false
})
});
stream.on("data", (key: string) => {
let s = key.split("/", 2);
if (s.length > 1)
keys.push(s[1]);
if (s.length > 1) keys.push(s[1]);
});
stream.on("end", () => yes(keys));
stream.on("error", no);
@ -477,8 +564,7 @@ export class CollectionQuery extends Query {
let seg = parts.shift();
d = data[seg];
if (d === undefined || d === null)
break; // Undefined/Null has no other fields!
if (d === undefined || d === null) break; // Undefined/Null has no other fields!
}
return d;
}
@ -513,21 +599,20 @@ export class CollectionQuery extends Query {
default:
throw new QueryError("Invalid where operation " + opStr);
}
})
});
}
return true;
}
async get(collection: string) {
if (!collection)
return [];
if (!collection) return [];
return new Promise<DocRes[]>((yes, no) => {
const stream = this.database.data.iterator({
...this.getStreamOptions(collection),
keyAsBuffer: false,
valueAsBuffer: true
})
});
let values: DocRes[] = [];
@ -535,16 +620,14 @@ export class CollectionQuery extends Query {
if (err) {
no(err);
stream.end(err => Logging.error(err));
}
else {
} else {
if (!key && !value) {
// END
Logging.debug("Checked all!")
Logging.debug("Checked all!");
yes(values);
} else {
let s = key.split("/", 2);
if (s.length <= 1)
return;
if (s.length <= 1) return;
const id = s[1];
@ -555,9 +638,8 @@ export class CollectionQuery extends Query {
id,
data
});
}
else {
stream.end((err) => err ? no(err) : yes(values))
} else {
stream.end(err => (err ? no(err) : yes(values)));
return;
}
}
@ -565,10 +647,10 @@ export class CollectionQuery extends Query {
stream.next(onValue);
}
}
}
};
stream.next(onValue);
})
});
}
checkChange(change: Change) {
@ -576,25 +658,30 @@ export class CollectionQuery extends Query {
}
firstSend(collection: string) {
return this.get(collection)
return this.get(collection);
}
public async collections() {
if (!this.session.root)
throw new QueryError("No Permission!");
if (!this.session.root) throw new QueryError("No Permission!");
return new Promise<string[]>((yes, no) => {
let keys = [];
const stream = this.database.data.createKeyStream({ keyAsBuffer: false })
const stream = this.database.data.createKeyStream({
keyAsBuffer: false
});
stream.on("data", (key: string) => keys.push(key.split("/")));
stream.on("end", () => yes(keys));
stream.on("error", no);
});
}
public async deleteCollection(collection: string, document: string, _b: LevelUpChain, collectionKey: string) {
if (!this.session.root)
throw new QueryError("No Permission!");
public async deleteCollection(
collection: string,
document: string,
_b: LevelUpChain,
collectionKey: string
) {
if (!this.session.root) throw new QueryError("No Permission!");
//TODO: Lock whole collection!
let batch = this.database.data.batch();
@ -615,8 +702,7 @@ export class CollectionQuery extends Query {
});
}
} finally {
if (batch)
batch.clear();
if (batch) batch.clear();
}
}
@ -629,4 +715,4 @@ export class QueryError extends Error {
constructor(message: string) {
super(message);
}
}
}

49
src/database/rules.ts
View File

@ -2,13 +2,15 @@ import Session from "./session";
import Logging from "@hibas123/nodelogging";
interface IRule<T> {
".write"?: T
".read"?: T
".write"?: T;
".read"?: T;
}
type IRuleConfig<T> = {
[segment: string]: IRuleConfig<T>;
} | IRule<T>;
type IRuleConfig<T> =
| IRule<T>
| {
[segment: string]: IRuleConfig<T>;
};
type IRuleRaw = IRuleConfig<string>;
type IRuleParsed = IRuleConfig<boolean>;
@ -17,17 +19,16 @@ const resolve = (value: any) => {
if (value === true) {
return true;
} else if (typeof value === "string") {
}
return undefined;
}
};
export class Rules {
rules: IRuleParsed;
constructor(private config: string) {
let parsed: IRuleRaw = JSON.parse(config);
const analyze = (raw: IRuleRaw) => {
const analyse = (raw: IRuleRaw) => {
let r: IRuleParsed = {};
if (raw[".read"]) {
@ -47,18 +48,25 @@ export class Rules {
}
for (let segment in raw) {
if (segment.startsWith("."))
continue;
if (segment.startsWith(".")) continue;
r[segment] = analyze(raw[segment]);
r[segment] = analyse(raw[segment]);
}
return r;
}
};
this.rules = analyze(parsed);
this.rules = analyse(parsed);
}
hasPermission(path: string[], session: Session): { read: boolean, write: boolean } {
hasPermission(
path: string[],
session: Session
): { read: boolean; write: boolean } {
if (session.root)
return {
read: true,
write: true
};
let read = this.rules[".read"] || false;
let write = this.rules[".write"] || false;
@ -77,22 +85,21 @@ export class Rules {
.find(e => {
switch (e) {
case "$uid":
if (segment === session.uid)
return true;
if (segment === session.uid) return true;
break;
}
return false;
})
});
rules = (k ? rules[k] : undefined) || rules[segment] || rules["*"];
if (rules) {
if (rules[".read"]) {
read = rules[".read"]
read = rules[".read"];
}
if (rules[".write"]) {
read = rules[".write"]
read = rules[".write"];
}
} else {
break;
@ -102,10 +109,10 @@ export class Rules {
return {
read: read as boolean,
write: write as boolean
}
};
}
toJSON() {
return this.config;
}
}
}

15
src/web/v1/index.ts
View File

@ -1,7 +1,11 @@
import * as Router from "koa-router";
import AdminRoute from "./admin";
import { DatabaseManager } from "../../database/database";
import { NotFoundError, NoPermissionError, BadRequestError } from "../helper/errors";
import {
NotFoundError,
NoPermissionError,
BadRequestError
} from "../helper/errors";
import Logging from "@hibas123/nodelogging";
import Session from "../../database/session";
import nanoid = require("nanoid");
@ -28,7 +32,7 @@ V1.post("/db/:database/query", async ctx => {
if (db.accesskey) {
if (!accesskey || accesskey !== db.accesskey) {
throw new NoPermissionError("");
throw new NoPermissionError("Invalid Access Key");
}
}
@ -36,7 +40,6 @@ V1.post("/db/:database/query", async ctx => {
let res = await verifyJWT(authkey, db.publickey);
if (!res || !res.uid) {
throw new BadRequestError("Invalid JWT");
return;
} else {
session.uid = res.uid;
}
@ -54,6 +57,6 @@ V1.post("/db/:database/query", async ctx => {
throw new BadRequestError(err.message);
}
throw err;
})
})
export default V1;
});
});
export default V1;