OpenServer/RealtimeDB-OLD
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improvements in collection storage. Breaks compatibility with previous versions!

Browse Source
This commit is contained in:
Fabian Stamm 2019-11-04 17:52:18 +01:00
parent 9f944549b6
commit aa2a846031
8 changed files with 131 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
package-lock.json generated
View File

@ -195,6 +195,15 @@
"integrity": "sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw==",
"dev": true
},
"@types/nanoid": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/@types/nanoid/-/nanoid-2.1.0.tgz",
"integrity": "sha512-xdkn/oRTA0GSNPLIKZgHWqDTWZsVrieKomxJBOQUK9YDD+zfSgmwD5t4WJYra5S7XyhTw7tfvwznW+pFexaepQ==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/node": {
"version": "12.12.5",
"resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.5.tgz",
@ -2645,9 +2654,9 @@
"optional": true
},
"nanoid": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-2.1.1.tgz",
"integrity": "sha512-0YbJdaL4JFoejIOoawgLcYValFGJ2iyUuVDIWL3g8Es87SSOWFbWdRUMV3VMSiyPs3SQ3QxCIxFX00q5DLkMCw=="
"version": "2.1.6",
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-2.1.6.tgz",
"integrity": "sha512-2NDzpiuEy3+H0AVtdt8LoFi7PnqkOnIzYmJQp7xsEU6VexLluHQwKREuiz57XaQC5006seIadPrIZJhyS2n7aw=="
},
"nanomatch": {
"version": "1.2.13",

4
package.json
View File

@ -21,8 +21,8 @@
"@types/koa-router": "^7.0.42",
"@types/leveldown": "^4.0.1",
"@types/levelup": "^3.1.1",
"@types/nanoid": "^2.1.0",
"@types/node": "^12.12.5",
"@types/shortid": "0.0.29",
"@types/ws": "^6.0.3",
"concurrently": "^5.0.0",
"nodemon": "^1.19.4",
@ -40,7 +40,7 @@
"koa-router": "^7.4.0",
"leveldown": "^5.4.1",
"levelup": "^4.3.2",
"shortid": "^2.2.15",
"nanoid": "^2.1.6",
"what-the-pack": "^2.0.3",
"ws": "^7.2.0"
}

29
src/connection.ts
View File

@ -4,7 +4,7 @@ import { DatabaseManager } from "./database/database";
import Logging from "@hibas123/logging";
import Query from "./database/query";
import Session from "./database/session";
import shortid = require("shortid");
import nanoid = require("nanoid");
import * as JWT from "jsonwebtoken";
@ -51,7 +51,7 @@ export class ConnectionManager {
const sendError = (error: string) => socket.send(JSON.stringify({ ns: "error_msg", data: error }));
const session = new Session(shortid());
const session = new Session(nanoid());
let query = new URLSearchParams(req.url.split("?").pop());
@ -104,6 +104,16 @@ export class ConnectionManager {
const noperm = new Error("No permisison!");
if (session.root) {
queryHandler.set("collections", (query, perm, data) => {
return query.getCollections();
})
queryHandler.set("delete-collection", (query, perm, data) => {
return query.deleteCollection();
})
}
queryHandler.set("keys", (query, perm, data) => {
if (!perm.read)
throw noperm;
@ -128,6 +138,12 @@ export class ConnectionManager {
return query.update(data);
})
queryHandler.set("push", (query, perm, data) => {
if (!perm.write)
throw noperm;
return query.push(data);
})
queryHandler.set("delete", (query, perm, data) => {
if (!perm.write)
throw noperm;
@ -138,7 +154,7 @@ export class ConnectionManager {
if (!perm.read)
throw noperm;
let subscriptionID = shortid.generate();
let subscriptionID = nanoid();
query.subscribe((data) => {
socket.send(JSON.stringify({ ns: "event", data: { id: subscriptionID, data } }));
@ -156,8 +172,6 @@ export class ConnectionManager {
//TODO: Handle case with no id, type, path
Logging.debug(`Request with id '${id}' from type '${type}' and path '${path}' with data`, data)
try {
if (!db)
throw new Error("Database not found!");
@ -170,7 +184,7 @@ export class ConnectionManager {
const perms = db.rules.hasPermission(path, session);
let res = await handler(query, perms, data);
if (res[StoreSym] !== undefined) {
if (res && typeof res === "object" && res[StoreSym] !== undefined) {
if (res[StoreSym])
stored.set(id, query);
else
@ -180,7 +194,8 @@ export class ConnectionManager {
}
}
} catch (err) {
Logging.error(err);
// Logging.error(err);
Logging.debug("Sending error:", err);
answer(id, err.message, true);
}
})

11
src/database/database.ts
View File

@ -54,12 +54,17 @@ export class Database {
private level = getLevelDB(this.name);
get data() {
return this.level;
return this.level.data;
}
get collections() {
return this.level.collection;
}
public rules: Rules;
public locks = new DocumentLock()
public collectionLocks = new DocumentLock()
public changes = new Map<string, Set<(change: Change) => void>>();
@ -89,8 +94,8 @@ export class Database {
}
async setRootKey(key: string) {
await Settings.setDatabaseAccessKey(this.name, key);
this.accesskey = key;
await Settings.setDatabaseRootKey(this.name, key);
this.rootkey = key;
}
async setPublicKey(key: string) {

4
src/database/lock.ts
View File

@ -3,6 +3,10 @@ export type Release = { release: () => void };
export default class DocumentLock {
private locks = new Map<string, (() => void)[]>();
getLocks() {
return Array.from(this.locks.keys())
}
async lock(collection: string = "", document: string = "") {
let key = collection + "/" + document;
let l = this.locks.get(key);

91
src/database/query.ts
View File

@ -1,11 +1,13 @@
import { Database, Change, ChangeTypes } from "./database";
import { resNull } from "../storage";
import shortid = require("shortid");
import nanoid = require("nanoid/generate");
import Logging from "@hibas123/nodelogging";
import * as MSGPack from "what-the-pack";
export const MP = MSGPack.initialize(2 ** 20);
const ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
const { encode, decode } = MP;
interface ISubscribeOptions {
@ -33,39 +35,28 @@ export default class Query {
}
private async resolve(path: string[], create = false): Promise<{ collection: string, document: string }> {
private async resolve(path: string[], create = false): Promise<{ collection: string, document: string, collectionKey: string }> {
path = [...path]; // Create modifiable copy
let collectionID: string = undefined;
let documentKey = undefined;
while (path.length > 0) {
let collectionName = path.shift();
let key = `${collectionID || ""}/${documentKey || ""}/${collectionName}`;
let lock = await this.database.locks.lock(collectionID, documentKey);
let documentKey = path.length % 2 === 0 ? path.pop() : undefined;
let key = path.join("/");
const lock = await this.database.collectionLocks.lock(key);
try {
collectionID = await this.database.data.get(key).then(r => r.toString()).catch(resNull);
if (!collectionID) {
if (create) {
collectionID = shortid.generate();
await this.database.data.put(key, Buffer.from(collectionID));
} else {
return { collection: null, document: null };
collectionID = await this.database.collections.get(key).then(r => r.toString()).catch(resNull);
if (!collectionID && create) {
collectionID = nanoid(ALPHABET, 32);
await this.database.collections.put(key, collectionID);
}
}
if (path.length > 0)
documentKey = path.shift();
else
documentKey = undefined;
} finally {
lock();
}
}
return {
collection: collectionID,
document: documentKey
document: documentKey,
collectionKey: key
};
}
@ -123,12 +114,12 @@ export default class Query {
let keys = [];
const stream = this.database.data.createKeyStream({
gt,
lt
lt,
keyAsBuffer: false
})
stream.on("data", (key: string | Buffer) => {
key = key.toString();
let s = key.split("/");
if (s.length === 2)
stream.on("data", (key: string) => {
let s = key.split("/", 2);
if (s.length > 1)
keys.push(s[1]);
});
stream.on("end", () => yes(keys));
@ -159,7 +150,7 @@ export default class Query {
}
public async push(value: any) {
let id = shortid.generate();
let id = nanoid(ALPHABET, 32);
let q = new Query(this.database, [...this.path, id], this.sender);
await q.set(value, {});
return id;
@ -189,7 +180,43 @@ export default class Query {
//TODO: Implement
}
subscription: {
public async getCollections() {
return new Promise<string[]>((yes, no) => {
let keys = [];
const stream = this.database.data.createKeyStream({ keyAsBuffer: false })
stream.on("data", (key: string) => keys.push(key.split("/")));
stream.on("end", () => yes(keys));
stream.on("error", no);
});
}
public async deleteCollection() {
const { collection, document, collectionKey } = await this.resolve(this.path);
if (document) {
throw new Error("There can be no document defined on this operation");
}
let batch = this.database.data.batch();
try {
if (collection) {
let keys = await this.keys();
for (let key in keys) {
batch.del(key);
}
await batch.write();
batch = undefined;
await this.database.collections.del(collectionKey);
}
} finally {
if (batch)
batch.clear();
}
}
private subscription: {
key: string;
type: Set<ChangeTypes>;
send: (data: Omit<Change, "sender">) => void;
@ -213,6 +240,8 @@ export default class Query {
type: new Set(type || [])
};
Logging.debug("Existing?", options.existing)
if (options.existing) {
if (document) {
send({

2
src/settings.ts
View File

@ -11,7 +11,7 @@ interface IDatabaseConfig {
class SettingComponent {
db = getLevelDB("_server");
db = getLevelDB("_server").data;
databaseLock = new Lock();
constructor() { }

27
src/storage.ts
View File

@ -9,8 +9,9 @@ import LevelDown, { LevelDown as LD } from "leveldown";
import { AbstractIterator } from "abstract-leveldown";
export type LevelDB = LU<LD, AbstractIterator<any, any>>;
export type DBSet = { data: LevelDB, collection: LevelDB };
const databases = new Map<string, LevelDB>();
const databases = new Map<string, DBSet>();
export function resNull(err): null {
if (!err.notFound)
@ -38,19 +39,31 @@ export async function deleteLevelDB(name: string) {
return;
let db = databases.get(name);
if (db && !db.isClosed())
await db.close()
if (db) {
if (db.data.isOpen())
await db.data.close()
if (db.collection.isOpen())
await db.collection.close()
}
//TODO make sure, that name doesn't make it possible to delete all databases :)
await rmRecursice("./databases/" + name);
}
export default function getLevelDB(name: string): LevelDB {
export default function getLevelDB(name: string): DBSet {
let db = databases.get(name);
if (!db || db.isClosed()) {
db = LevelUp(LevelDown("./databases/" + name));
databases.set(name, db);
if (!db) {
if (!fs.existsSync("./databases/" + name)) {
fs.mkdirSync("./databases/" + name);
}
}
db = {
data: db && db.data.isOpen() ? db.data : LevelUp(LevelDown("./databases/" + name + "/data")),
collection: db && db.collection.isOpen() ? db.collection : LevelUp(LevelDown("./databases/" + name + "/collection"))
}
databases.set(name, db);
return db;
}