diff --git a/package-lock.json b/package-lock.json index 13f48dc..353b4f2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -195,6 +195,15 @@ "integrity": "sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw==", "dev": true }, + "@types/nanoid": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/nanoid/-/nanoid-2.1.0.tgz", + "integrity": "sha512-xdkn/oRTA0GSNPLIKZgHWqDTWZsVrieKomxJBOQUK9YDD+zfSgmwD5t4WJYra5S7XyhTw7tfvwznW+pFexaepQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/node": { "version": "12.12.5", "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.5.tgz", @@ -2645,9 +2654,9 @@ "optional": true }, "nanoid": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-2.1.1.tgz", - "integrity": "sha512-0YbJdaL4JFoejIOoawgLcYValFGJ2iyUuVDIWL3g8Es87SSOWFbWdRUMV3VMSiyPs3SQ3QxCIxFX00q5DLkMCw==" + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-2.1.6.tgz", + "integrity": "sha512-2NDzpiuEy3+H0AVtdt8LoFi7PnqkOnIzYmJQp7xsEU6VexLluHQwKREuiz57XaQC5006seIadPrIZJhyS2n7aw==" }, "nanomatch": { "version": "1.2.13", diff --git a/package.json b/package.json index 6d98c2a..ecc7907 100644 --- a/package.json +++ b/package.json @@ -21,8 +21,8 @@ "@types/koa-router": "^7.0.42", "@types/leveldown": "^4.0.1", "@types/levelup": "^3.1.1", + "@types/nanoid": "^2.1.0", "@types/node": "^12.12.5", - "@types/shortid": "0.0.29", "@types/ws": "^6.0.3", "concurrently": "^5.0.0", "nodemon": "^1.19.4", @@ -40,8 +40,8 @@ "koa-router": "^7.4.0", "leveldown": "^5.4.1", "levelup": "^4.3.2", - "shortid": "^2.2.15", + "nanoid": "^2.1.6", "what-the-pack": "^2.0.3", "ws": "^7.2.0" } -} +} \ No newline at end of file diff --git a/src/connection.ts b/src/connection.ts index 15e0de4..79a85aa 100644 --- a/src/connection.ts +++ b/src/connection.ts @@ -4,7 +4,7 @@ import { DatabaseManager } from "./database/database"; import Logging from "@hibas123/logging"; import Query from "./database/query"; import Session from "./database/session"; -import shortid = require("shortid"); +import nanoid = require("nanoid"); import * as JWT from "jsonwebtoken"; @@ -51,7 +51,7 @@ export class ConnectionManager { const sendError = (error: string) => socket.send(JSON.stringify({ ns: "error_msg", data: error })); - const session = new Session(shortid()); + const session = new Session(nanoid()); let query = new URLSearchParams(req.url.split("?").pop()); @@ -104,6 +104,16 @@ export class ConnectionManager { const noperm = new Error("No permisison!"); + if (session.root) { + queryHandler.set("collections", (query, perm, data) => { + return query.getCollections(); + }) + + queryHandler.set("delete-collection", (query, perm, data) => { + return query.deleteCollection(); + }) + } + queryHandler.set("keys", (query, perm, data) => { if (!perm.read) throw noperm; @@ -128,6 +138,12 @@ export class ConnectionManager { return query.update(data); }) + queryHandler.set("push", (query, perm, data) => { + if (!perm.write) + throw noperm; + return query.push(data); + }) + queryHandler.set("delete", (query, perm, data) => { if (!perm.write) throw noperm; @@ -138,7 +154,7 @@ export class ConnectionManager { if (!perm.read) throw noperm; - let subscriptionID = shortid.generate(); + let subscriptionID = nanoid(); query.subscribe((data) => { socket.send(JSON.stringify({ ns: "event", data: { id: subscriptionID, data } })); @@ -156,8 +172,6 @@ export class ConnectionManager { //TODO: Handle case with no id, type, path Logging.debug(`Request with id '${id}' from type '${type}' and path '${path}' with data`, data) - - try { if (!db) throw new Error("Database not found!"); @@ -170,7 +184,7 @@ export class ConnectionManager { const perms = db.rules.hasPermission(path, session); let res = await handler(query, perms, data); - if (res[StoreSym] !== undefined) { + if (res && typeof res === "object" && res[StoreSym] !== undefined) { if (res[StoreSym]) stored.set(id, query); else @@ -180,7 +194,8 @@ export class ConnectionManager { } } } catch (err) { - Logging.error(err); + // Logging.error(err); + Logging.debug("Sending error:", err); answer(id, err.message, true); } }) diff --git a/src/database/database.ts b/src/database/database.ts index cf44152..d1d45a5 100644 --- a/src/database/database.ts +++ b/src/database/database.ts @@ -54,12 +54,17 @@ export class Database { private level = getLevelDB(this.name); get data() { - return this.level; + return this.level.data; + } + + get collections() { + return this.level.collection; } public rules: Rules; public locks = new DocumentLock() + public collectionLocks = new DocumentLock() public changes = new Map void>>(); @@ -89,8 +94,8 @@ export class Database { } async setRootKey(key: string) { - await Settings.setDatabaseAccessKey(this.name, key); - this.accesskey = key; + await Settings.setDatabaseRootKey(this.name, key); + this.rootkey = key; } async setPublicKey(key: string) { diff --git a/src/database/lock.ts b/src/database/lock.ts index dc1e825..4388dd7 100644 --- a/src/database/lock.ts +++ b/src/database/lock.ts @@ -3,6 +3,10 @@ export type Release = { release: () => void }; export default class DocumentLock { private locks = new Map void)[]>(); + getLocks() { + return Array.from(this.locks.keys()) + } + async lock(collection: string = "", document: string = "") { let key = collection + "/" + document; let l = this.locks.get(key); diff --git a/src/database/query.ts b/src/database/query.ts index 62c9825..c7d39d2 100644 --- a/src/database/query.ts +++ b/src/database/query.ts @@ -1,11 +1,13 @@ import { Database, Change, ChangeTypes } from "./database"; import { resNull } from "../storage"; -import shortid = require("shortid"); +import nanoid = require("nanoid/generate"); import Logging from "@hibas123/nodelogging"; import * as MSGPack from "what-the-pack"; export const MP = MSGPack.initialize(2 ** 20); +const ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + const { encode, decode } = MP; interface ISubscribeOptions { @@ -33,39 +35,28 @@ export default class Query { } - private async resolve(path: string[], create = false): Promise<{ collection: string, document: string }> { + private async resolve(path: string[], create = false): Promise<{ collection: string, document: string, collectionKey: string }> { path = [...path]; // Create modifiable copy let collectionID: string = undefined; - let documentKey = undefined; - while (path.length > 0) { - let collectionName = path.shift(); - let key = `${collectionID || ""}/${documentKey || ""}/${collectionName}`; - let lock = await this.database.locks.lock(collectionID, documentKey); - try { - collectionID = await this.database.data.get(key).then(r => r.toString()).catch(resNull); + let documentKey = path.length % 2 === 0 ? path.pop() : undefined; + let key = path.join("/"); - if (!collectionID) { - if (create) { - collectionID = shortid.generate(); - await this.database.data.put(key, Buffer.from(collectionID)); - } else { - return { collection: null, document: null }; - } - } + const lock = await this.database.collectionLocks.lock(key); - - if (path.length > 0) - documentKey = path.shift(); - else - documentKey = undefined; - } finally { - lock(); + try { + collectionID = await this.database.collections.get(key).then(r => r.toString()).catch(resNull); + if (!collectionID && create) { + collectionID = nanoid(ALPHABET, 32); + await this.database.collections.put(key, collectionID); } + } finally { + lock(); } return { collection: collectionID, - document: documentKey + document: documentKey, + collectionKey: key }; } @@ -123,12 +114,12 @@ export default class Query { let keys = []; const stream = this.database.data.createKeyStream({ gt, - lt + lt, + keyAsBuffer: false }) - stream.on("data", (key: string | Buffer) => { - key = key.toString(); - let s = key.split("/"); - if (s.length === 2) + stream.on("data", (key: string) => { + let s = key.split("/", 2); + if (s.length > 1) keys.push(s[1]); }); stream.on("end", () => yes(keys)); @@ -159,7 +150,7 @@ export default class Query { } public async push(value: any) { - let id = shortid.generate(); + let id = nanoid(ALPHABET, 32); let q = new Query(this.database, [...this.path, id], this.sender); await q.set(value, {}); return id; @@ -189,7 +180,43 @@ export default class Query { //TODO: Implement } - subscription: { + public async getCollections() { + return new Promise((yes, no) => { + let keys = []; + const stream = this.database.data.createKeyStream({ keyAsBuffer: false }) + stream.on("data", (key: string) => keys.push(key.split("/"))); + stream.on("end", () => yes(keys)); + stream.on("error", no); + }); + } + + public async deleteCollection() { + const { collection, document, collectionKey } = await this.resolve(this.path); + + if (document) { + throw new Error("There can be no document defined on this operation"); + } + + let batch = this.database.data.batch(); + try { + if (collection) { + let keys = await this.keys(); + for (let key in keys) { + batch.del(key); + } + await batch.write(); + batch = undefined; + await this.database.collections.del(collectionKey); + } + } finally { + if (batch) + batch.clear(); + } + } + + + + private subscription: { key: string; type: Set; send: (data: Omit) => void; @@ -213,6 +240,8 @@ export default class Query { type: new Set(type || []) }; + Logging.debug("Existing?", options.existing) + if (options.existing) { if (document) { send({ diff --git a/src/settings.ts b/src/settings.ts index 668c4d7..fb52ec1 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -11,7 +11,7 @@ interface IDatabaseConfig { class SettingComponent { - db = getLevelDB("_server"); + db = getLevelDB("_server").data; databaseLock = new Lock(); constructor() { } diff --git a/src/storage.ts b/src/storage.ts index f4dbc86..f9c8ca5 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -9,8 +9,9 @@ import LevelDown, { LevelDown as LD } from "leveldown"; import { AbstractIterator } from "abstract-leveldown"; export type LevelDB = LU>; +export type DBSet = { data: LevelDB, collection: LevelDB }; -const databases = new Map(); +const databases = new Map(); export function resNull(err): null { if (!err.notFound) @@ -38,19 +39,31 @@ export async function deleteLevelDB(name: string) { return; let db = databases.get(name); - if (db && !db.isClosed()) - await db.close() + if (db) { + if (db.data.isOpen()) + await db.data.close() + if (db.collection.isOpen()) + await db.collection.close() + } //TODO make sure, that name doesn't make it possible to delete all databases :) await rmRecursice("./databases/" + name); } -export default function getLevelDB(name: string): LevelDB { +export default function getLevelDB(name: string): DBSet { let db = databases.get(name); - if (!db || db.isClosed()) { - db = LevelUp(LevelDown("./databases/" + name)); - databases.set(name, db); + if (!db) { + if (!fs.existsSync("./databases/" + name)) { + fs.mkdirSync("./databases/" + name); + } } + + db = { + data: db && db.data.isOpen() ? db.data : LevelUp(LevelDown("./databases/" + name + "/data")), + collection: db && db.collection.isOpen() ? db.collection : LevelUp(LevelDown("./databases/" + name + "/collection")) + } + + databases.set(name, db); return db; } \ No newline at end of file