Version bump

Extending Admin Interface
Adding cleanup procedure, that clears undeleted collection data

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@hibas123/realtimedb",
-  "version": "2.0.1-0",
+  "version": "2.0.0-beta.4",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -225,12 +225,6 @@
         "@types/mime": "*"
       }
     },
-    "@types/shortid": {
-      "version": "0.0.29",
-      "resolved": "https://registry.npmjs.org/@types/shortid/-/shortid-0.0.29.tgz",
-      "integrity": "sha1-gJPuBBam4r8qpjOBCRFLP7/6Dps=",
-      "dev": true
-    },
     "@types/ws": {
       "version": "6.0.3",
       "resolved": "https://registry.npmjs.org/@types/ws/-/ws-6.0.3.tgz",
@@ -3290,14 +3284,6 @@
       "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=",
       "dev": true
     },
-    "shortid": {
-      "version": "2.2.15",
-      "resolved": "https://registry.npmjs.org/shortid/-/shortid-2.2.15.tgz",
-      "integrity": "sha512-5EaCy2mx2Jgc/Fdn9uuDuNIIfWBpzY4XIlhoqtXF6qsf+/+SGZ+FxDdX/ZsMZiWupIWNqAEmiNY4RC+LSmCeOw==",
-      "requires": {
-        "nanoid": "^2.1.0"
-      }
-    },
     "signal-exit": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hibas123/realtimedb",
-  "version": "2.0.0-beta.3",
+  "version": "2.0.0-beta.8",
   "description": "",
   "main": "lib/index.js",
   "private": true,
@@ -30,7 +30,6 @@
     "typescript": "^3.6.4"
   },
   "dependencies": {
-    "@hibas123/logging": "^2.1.1",
     "@hibas123/nodelogging": "^2.1.1",
     "@hibas123/utils": "^2.1.1",
     "dotenv": "^8.2.0",

src/config.ts

@@ -1,5 +1,6 @@
 import Logging from "@hibas123/nodelogging";
 import * as dotenv from "dotenv";
+import { LoggingTypes } from "@hibas123/logging";
 dotenv.config()
 
 
@@ -17,4 +18,8 @@ const config: IConfig = {
    dev: (process.env.DEV || "").toLowerCase() === "true"
 }
 
+if (config.dev) {
+   Logging.logLevel = LoggingTypes.Log;
+}
+
 export default config;

src/connection.ts

@@ -1,43 +1,12 @@
+import Logging from "@hibas123/nodelogging";
+import { IncomingMessage, Server } from "http";
 import * as WebSocket from "ws";
-import { Server, IncomingMessage } from "http";
+import { DatabaseManager, IQuery, ITypedQuery } from "./database/database";
-import { DatabaseManager } from "./database/database";
+import { CollectionQuery, DocumentQuery } from "./database/query";
-import Logging from "@hibas123/logging";
-import { Query, CollectionQuery, DocumentQuery } from "./database/query";
 import Session from "./database/session";
+import { verifyJWT } from "./helper/jwt";
 import nanoid = require("nanoid");
 
-import * as JWT from "jsonwebtoken";
-
-async function verifyJWT(token: string, publicKey: string) {
-   return new Promise<any | undefined>((yes) => {
-      JWT.verify(token, publicKey, (err, decoded) => {
-         if (err)
-            yes(undefined);
-         else
-            yes(decoded);
-      })
-   })
-}
-
-const StoreSym = Symbol("store");
-function StoreQuery(result?: any) {
-   return {
-      [StoreSym]: true,
-      result
-   }
-}
-
-function DeleteQuery(result?: any) {
-   return {
-      [StoreSym]: false,
-      result
-   }
-}
-
-import { URLSearchParams } from "url";
-
-type QueryTypes = "keys" | "get" | "set" | "update" | "delete" | "push" | "subscribe" | "unsubscribe";
-
 export class ConnectionManager {
    static server: WebSocket.Server;
 
@@ -50,10 +19,9 @@ export class ConnectionManager {
       Logging.log("New Connection:");
       const sendError = (error: string) => socket.send(JSON.stringify({ ns: "error_msg", data: error }));
 
-
       const session = new Session(nanoid());
 
-      let query = new URLSearchParams(req.url.split("?").pop());
+      const query = new URL(req.url, "http://localhost").searchParams;
 
       const database = query.get("database");
       const db = DatabaseManager.getDatabase(database);
@@ -92,53 +60,34 @@ export class ConnectionManager {
          }
       }
 
-      const stored = new Map<string, Query>();
       const answer = (id: string, data: any, error: boolean = false) => {
          socket.send(JSON.stringify({ ns: "message", data: { id, error, data } }));
       }
 
       const handler = new Map<string, ((data: any) => void)>();
-      type QueryData = { id: string, type: QueryTypes, path: string[], data: any, options: any };
 
-      handler.set("query", async ({ id, type, path, data }: QueryData) => {
+      handler.set("v2", async ({ id, query }: { id: string, query: IQuery }) => db.run(query, session)
-         //TODO: Handle case with no id, type, path
+         .then(res => answer(id, res))
-         Logging.debug(`Request with id '${id}' from type '${type}' and path '${path.join("/")}' with data`, data)
+         .catch(err => answer(id, undefined, err)));
 
-         try {
-            if (!db)
-               throw new Error("Database not found!");
-            else {
-               let isDoc = path.length % 2 === 0;
-               let handler = isDoc ? documentHandler.get(type) : collectionHandler.get(type);
 
-               if (!handler && session.root)
+      const SnapshotMap = new Map<string, string>();
-                  handler = rootHandler.get(type);
+      handler.set("snapshot", async ({ id, query }: { id: string, query: ITypedQuery<"snapshot"> }) => {
+         db.snapshot(query, session, (data => {
+            socket.send(JSON.stringify({
+               ns: "snapshot", data: { id, data }
+            }));
+         })).then(s => {
+            answer(id, s.snaphot);
+            SnapshotMap.set(id, s.id);
+         }).catch(err => answer(id, undefined, err));
+      })
 
-               if (!handler)
+      handler.set("unsubscribe", async ({ id }) => {
-                  throw new Error("Invalid Request!");
+         let i = SnapshotMap.get(id);
-
+         if (i) {
-               let query = db.getQuery(path || [], session.sessionid, isDoc ? "document" : "collection");
+            db.unsubscribe(i, session);
-               let res = await handler({
+            SnapshotMap.delete(i);
-                  id,
-                  data,
-                  socket,
-                  query: query as any // We know it is the right one
-               })
-
-               if (res && typeof res === "object" && res[StoreSym] !== undefined) {
-                  if (res[StoreSym])
-                     stored.set(id, query);
-                  else
-                     stored.delete(id);
-
-                  res = res.result;
-               }
-               answer(id, res);
-            }
-         } catch (err) {
-            // Logging.error(err);
-            Logging.debug("Sending error:", err);
-            answer(id, err.message, true);
          }
       })
 
@@ -156,102 +105,12 @@ export class ConnectionManager {
       })
 
       socket.on("close", () => {
-         Logging.log(`${session.sessionid} has disconnected!`);
+         Logging.log(`${session.id} has disconnected!`);
-         Logging.debug("Clearing stored:", stored);
+         session.queries.forEach((query: DocumentQuery | CollectionQuery) => {
-         stored.forEach(query => (query as DocumentQuery | CollectionQuery).unsubscribe());
+            query.unsubscribe();
-         stored.clear();
+         })
+         session.queries.clear();
          socket.removeAllListeners();
       })
    }
 }
-
-type QueryHandler<T extends Query> = (api: {
-   id: string;
-   query: T;
-   // storedQuery(id: string): T  | undefined;   
-   socket: WebSocket;
-   data: any;
-}) => any;
-
-const NoPermissionError = new Error("No permisison!");
-
-const rootHandler = new Map<string, QueryHandler<Query>>();
-rootHandler.set("collections", ({ query }) => {
-   return CollectionQuery.fromQuery(query).collections();
-})
-
-rootHandler.set("delete-collection", ({ query }) => {
-   return CollectionQuery.fromQuery(query).deleteCollection();
-})
-
-
-const documentHandler = new Map<string, QueryHandler<DocumentQuery>>();
-documentHandler.set("get", ({ query }) => {
-   return query.get();
-})
-
-documentHandler.set("set", ({ query, data }) => {
-   return query.set(data, {});
-})
-
-documentHandler.set("update", ({ query, data }) => {
-   return query.update(data);
-})
-
-documentHandler.set("delete", ({ query }) => {
-   return query.delete();
-})
-
-documentHandler.set("snapshot", async ({ query, data, id, socket }) => {
-   let res = await query.snapshot((data) => {
-      socket.send(JSON.stringify({
-         ns: "snapshot", data: { id, data }
-      }));
-   });
-
-   return StoreQuery(res);
-})
-
-documentHandler.set("unsubscribe", async ({ query }) => {
-   query.unsubscribe();
-   return DeleteQuery(true);
-})
-
-
-const collectionHandler = new Map<string, QueryHandler<CollectionQuery>>();
-collectionHandler.set("keys", ({ query }) => {
-   return query.keys();
-})
-
-
-collectionHandler.set("add", ({ query, data }) => {
-   return query.add(data);
-})
-
-collectionHandler.set("get", ({ query, data }) => {
-   if (data.where)
-      query.where = data.where;
-   if (data.limit)
-      query.limit = data.limit;
-   return query.get();
-})
-
-collectionHandler.set("snapshot", async ({ query, id, socket, data }) => {
-   if (data.where)
-      query.where = data.where;
-   if (data.limit)
-      query.limit = data.limit;
-
-   let res = await query.snapshot((data) => {
-      socket.send(JSON.stringify({
-         ns: "snapshot", data: { id, data }
-      }));
-   });
-
-   return StoreQuery(res);
-})
-
-collectionHandler.set("unsubscribe", async ({ query }) => {
-   query.unsubscribe();
-   return DeleteQuery(true);
-})

src/database/database.ts

@@ -2,7 +2,28 @@ import { Rules } from "./rules";
 import Settings from "../settings";
 import getLevelDB, { LevelDB, deleteLevelDB } from "../storage";
 import DocumentLock from "./lock";
-import { DocumentQuery, CollectionQuery, Query } from "./query";
+import { DocumentQuery, CollectionQuery, Query, QueryError } from "./query";
+import Logging from "@hibas123/nodelogging";
+import Session from "./session";
+import nanoid = require("nanoid");
+
+type IWriteQueries = "set" | "update" | "delete" | "add";
+type ICollectionQueries = "get" | "add" | "keys" | "delete-collection" | "list";
+type IDocumentQueries = "get" | "set" | "update" | "delete";
+
+export interface ITypedQuery<T> {
+   path: string[];
+   type: T;
+   data?: any;
+   options?: any;
+}
+
+interface ITransaction {
+   queries: ITypedQuery<IWriteQueries>[];
+}
+
+export type IQuery = ITypedQuery<ICollectionQueries | IDocumentQueries>;
+
 
 export class DatabaseManager {
    static databases = new Map<string, Database>();
@@ -104,16 +125,206 @@ export class Database {
    }
 
 
-   getQuery(path: string[], sender: string, type: "document" | "collection" | "any") {
+   getQuery(path: string[], session: Session, type: "document" | "collection" | "any") {
       if (type === "document")
-         return new DocumentQuery(this, path, sender);
+         return new DocumentQuery(this, path, session);
       else if (type === "collection")
-         return new CollectionQuery(this, path, sender);
+         return new CollectionQuery(this, path, session);
       else
-         return new Query(this, path, sender);
+         return new Query(this, path, session);
+   }
+
+   private validate(query: ITypedQuery<any>) {
+      const inv = new QueryError("Malformed query!");
+      if (!query || typeof query !== "object")
+         throw inv;
+
+      if (!query.type)
+         throw inv;
+
+      if (!query.path)
+         throw inv;
+   }
+
+   async run(query: IQuery, session: Session) {
+      this.validate(query);
+      const isCollection = query.path.length % 2 === 1;
+      if (isCollection) {
+         const q = new CollectionQuery(this, query.path, session);
+         let type = query.type as ICollectionQueries;
+         switch (type) {
+            case "add":
+               return q.add(query.data);
+            case "get":
+               const limit = (query.options || {}).limit;
+               if (limit)
+                  q.limit = limit;
+               const where = (query.options || {}).where;
+               if (where)
+                  q.where = where;
+               return q.get();
+            case "keys":
+               return q.keys();
+            case "list":
+               return q.collections();
+            case "delete-collection":
+               return q.deleteCollection();
+            default:
+               return Promise.reject(new Error("Invalid query!"));
+         }
+      } else {
+         const q = new DocumentQuery(this, query.path, session);
+         let type = query.type as IDocumentQueries;
+         switch (type) {
+            case "get":
+               return q.get();
+            case "set":
+               return q.set(query.data, query.options || {});
+            case "update":
+               return q.update(query.data);
+            case "delete":
+               return q.delete();
+            default:
+               return Promise.reject(new Error("Invalid query!"));
+         }
+      }
+   }
+
+   async snapshot(query: ITypedQuery<"snapshot">, session: Session, onchange: (change: any) => void) {
+      this.validate(query);
+
+      const isCollection = query.path.length % 2 === 1;
+      let q: DocumentQuery | CollectionQuery;
+      if (isCollection) {
+         q = new CollectionQuery(this, query.path, session);
+         const limit = (query.options || {}).limit;
+         if (limit)
+            q.limit = limit;
+         const where = (query.options || {}).where;
+         if (where)
+            q.where = where;
+      } else {
+         q = new DocumentQuery(this, query.path, session);
+      }
+
+      const id = nanoid(16);
+      session.queries.set(id, q);
+      return {
+         id,
+         snaphot: await q.snapshot(onchange)
+      };
+   }
+
+   async unsubscribe(id: string, session: Session) {
+      let query: CollectionQuery | DocumentQuery = session.queries.get(id) as any;
+      if (query) {
+         query.unsubscribe();
+         session.queries.delete(id);
+      }
    }
 
    async stop() {
       await this.data.close();
    }
+
+   public async runCleanup() {
+      const should = await new Promise<Set<string>>((yes, no) => {
+         const stream = this.collections.iterator({
+            keyAsBuffer: false,
+            valueAsBuffer: false
+         })
+
+         const collections = new Set<string>();
+         const onValue = (err: Error, key: string, value: string) => {
+            if (err) {
+               Logging.error(err);
+               stream.end((err) => Logging.error(err))
+               no(err);
+            }
+
+            if (!key && !value) {
+               yes(collections);
+            } else {
+               collections.add(value)
+               stream.next(onValue);
+            }
+         }
+
+         stream.next(onValue);
+      })
+
+      const existing = await new Promise<Set<string>>((yes, no) => {
+         const stream = this.data.iterator({
+            keyAsBuffer: false,
+            values: false
+         })
+
+         const collections = new Set<string>();
+         const onValue = (err: Error, key: string, value: Buffer) => {
+            if (err) {
+               Logging.error(err);
+               stream.end((err) => Logging.error(err))
+               no(err);
+            }
+
+            if (!key && !value) {
+               yes(collections);
+            } else {
+               let coll = key.split("/")[0];
+               collections.add(coll)
+               stream.next(onValue);
+            }
+         }
+
+         stream.next(onValue);
+      })
+
+      const toDelete = new Set<string>();
+      existing.forEach(collection => {
+         if (!should.has(collection))
+            toDelete.add(collection);
+      })
+
+      for (let collection of toDelete) {
+         const batch = this.data.batch();
+
+         let gt = Buffer.from(collection + "/ ");
+         gt[gt.length - 1] = 0;
+
+         let lt = Buffer.alloc(gt.length);
+         lt.set(gt);
+         lt[gt.length - 1] = 0xFF;
+
+         await new Promise<void>((yes, no) => {
+            const stream = this.data.iterator({
+               keyAsBuffer: false,
+               values: false,
+               gt,
+               lt
+            })
+
+            const onValue = (err: Error, key: string, value: Buffer) => {
+               if (err) {
+                  Logging.error(err);
+                  stream.end((err) => Logging.error(err))
+                  no(err);
+               }
+
+               if (!key && !value) {
+                  yes();
+               } else {
+                  batch.del(key);
+                  stream.next(onValue);
+               }
+            }
+
+            stream.next(onValue);
+         })
+
+         await batch.write();
+      }
+
+      return Array.from(toDelete.values());
+   }
+
 }

src/database/query.ts

@@ -3,6 +3,7 @@ import { resNull } from "../storage";
 import nanoid = require("nanoid/generate");
 import Logging from "@hibas123/nodelogging";
 import * as MSGPack from "what-the-pack";
+import Session from "./session";
 
 export const MP = MSGPack.initialize(2 ** 20);
 
@@ -10,10 +11,6 @@ const ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
 
 const { encode, decode } = MP;
 
-interface ISubscribeOptions {
-   existing: boolean;
-}
-
 export class Query {
    /**
     * Returns true if the path only contains valid characters and false if it doesn't
@@ -23,12 +20,12 @@ export class Query {
       return path.every(e => (e.match(/[^a-zA-Z0-9_\-\<\>]/g) || []).length === 0);
    }
 
-   constructor(protected database: Database, protected path: string[], protected sender: string) {
+   constructor(protected database: Database, protected path: string[], protected session: Session) {
       if (path.length > 10) {
-         throw new Error("Path is to long. Path is only allowed to be 10 Layers deep!");
+         throw new QueryError("Path is to long. Path is only allowed to be 10 Layers deep!");
       }
       if (!this.validatePath(path)) {
-         throw new Error("Path can only contain a-z A-Z 0-9  '-' '-' '<' and '>' ");
+         throw new QueryError("Path can only contain a-z A-Z 0-9  '-' '-' '<' and '>' ");
       }
    }
 
@@ -73,7 +70,7 @@ export class Query {
          type,
          document,
          data,
-         sender: this.sender
+         sender: this.session.id
       }
 
       let s = this.database.changes.get(this.getKey(collection, document))
@@ -86,8 +83,8 @@ export class Query {
 
    }
 
-   protected static getConstructorParams(query: Query): [Database, string[], string] {
+   protected static getConstructorParams(query: Query): [Database, string[], Session] {
-      return [query.database, query.path, query.sender];
+      return [query.database, query.path, query.session];
    }
 }
 
@@ -98,8 +95,8 @@ interface UpdateData {
    }
 }
 export class DocumentQuery extends Query {
-   constructor(database: Database, path: string[], sender: string) {
+   constructor(database: Database, path: string[], session: Session) {
-      super(database, path, sender);
+      super(database, path, session);
       this.onChange = this.onChange.bind(this);
    }
 
@@ -118,11 +115,11 @@ export class DocumentQuery extends Query {
          return this.delete();
       let { collection, document } = await this.resolve(this.path, true);
       if (!collection) {
-         throw new Error("There must be a collection!")
+         throw new QueryError("There must be a collection!")
       }
 
       if (!document) {
-         throw new Error("There must be a document key!")
+         throw new QueryError("There must be a document key!")
       }
 
       const lock = await this.database.locks.lock(collection, document);
@@ -138,11 +135,11 @@ export class DocumentQuery extends Query {
    public async update(updateData: UpdateData) {
       let { collection, document } = await this.resolve(this.path, true);
       if (!collection) {
-         throw new Error("There must be a collection!")
+         throw new QueryError("There must be a collection!")
       }
 
       if (!document) {
-         throw new Error("There must be a document key!")
+         throw new QueryError("There must be a document key!")
       }
 
       // Logging.debug(updateData);
@@ -179,7 +176,7 @@ export class DocumentQuery extends Query {
                   if (d[last] === undefined || d[last] === null)
                      d[last] = toUpdate.value;
                   else if (typeof d[last] !== "number") {
-                     throw new Error("Field is no number!");
+                     throw new QueryError("Field is no number!");
                   } else {
                      d[last] += toUpdate.value;
                   }
@@ -193,11 +190,11 @@ export class DocumentQuery extends Query {
                   else if (Array.isArray(d[last])) {
                      d[last].push(toUpdate.value);
                   } else {
-                     throw new Error("Field is not array!");
+                     throw new QueryError("Field is not array!");
                   }
                   break;
                default:
-                  throw new Error("Invalid update type: " + toUpdate.type);
+                  throw new QueryError("Invalid update type: " + toUpdate.type);
             }
          }
 
@@ -214,11 +211,11 @@ export class DocumentQuery extends Query {
       let { collection, document } = await this.resolve(this.path);
 
       if (!collection) {
-         throw new Error("There must be a collection!")
+         throw new QueryError("There must be a collection!")
       }
 
       if (!document) {
-         throw new Error("There must be a document key!")
+         throw new QueryError("There must be a document key!")
       }
 
       const lock = await this.database.locks.lock(collection, document);
@@ -238,7 +235,7 @@ export class DocumentQuery extends Query {
 
    async snapshot(onChange: (change: DocRes & { type: ChangeTypes }) => void) {
       if (this.subscription)
-         throw new Error("This query is already subscribed!");
+         throw new QueryError("This query is already subscribed!");
       let { collection, document } = await this.resolve(this.path);
 
       let data = await this.getDoc(collection, document);
@@ -295,12 +292,15 @@ type WhereFilterOp =
    | 'in'
    | 'array-contains-any';
 
-interface IQueryWhere {
+interface IQueryWhereVerbose {
    fieldPath: FieldPath,
    opStr: WhereFilterOp,
    value: any
 }
 
+type IQueryWhereArray = [FieldPath, WhereFilterOp, any];
+
+type IQueryWhere = IQueryWhereArray | IQueryWhereVerbose;
 
 interface DocRes {
    id: string;
@@ -308,18 +308,39 @@ interface DocRes {
 }
 
 export class CollectionQuery extends Query {
-   constructor(database: Database, path: string[], sender: string) {
+   constructor(database: Database, path: string[], session: Session) {
-      super(database, path, sender);
+      super(database, path, session);
       this.onChange = this.onChange.bind(this);
    }
 
 
-   public where: IQueryWhere[] = [];
+   private _where: IQueryWhereArray[] = [];
+   public set where(value: IQueryWhere[]) {
+      const invalidWhere = new QueryError("Invalid Where");
+      if (!Array.isArray(value))
+         throw invalidWhere;
+      let c = [];
+      this._where = value.map(cond => {
+         Logging.debug("Query Condition", cond);
+         if (Array.isArray(cond)) {
+            if (cond.length !== 3)
+               throw invalidWhere;
+            return cond;
+         } else {
+            if (cond && typeof cond === "object" && "fieldPath" in cond && "opStr" in cond && "value" in cond) {
+               return [cond.fieldPath, cond.opStr, cond.value];
+            } else {
+               throw invalidWhere;
+            }
+         }
+      })
+   }
+
    public limit: number = -1;
 
    public async add(value: any) {
       let id = nanoid(ALPHABET, 32);
-      let q = new DocumentQuery(this.database, [...this.path, id], this.sender);
+      let q = new DocumentQuery(this.database, [...this.path, id], this.session);
       await q.set(value, {});
       return id;
    }
@@ -341,9 +362,9 @@ export class CollectionQuery extends Query {
    public async keys() {
       let { collection, document } = await this.resolve(this.path);
       if (document)
-         throw new Error("Keys only works on collections!");
+         throw new QueryError("Keys only works on collections!");
       if (!collection)
-         throw new Error("There must be a collection");
+         return []
 
       return new Promise<string[]>((yes, no) => {
          let keys = [];
@@ -375,31 +396,34 @@ export class CollectionQuery extends Query {
    }
 
    private fitsWhere(data: any): boolean {
-      if (this.where.length > 0) {
+      if (this._where.length > 0) {
-         return this.where.every(where => {
+         return this._where.every(([fieldPath, opStr, value]) => {
-            let val = this.getFieldValue(data, where.fieldPath);
+            let val = this.getFieldValue(data, fieldPath);
-            switch (where.opStr) {
+            switch (opStr) {
                case "<":
-                  return val < where.value;
+                  return val < value;
                case "<=":
-                  return val <= where.value;
+                  return val <= value;
                case "==":
-                  return val == where.value;
+                  return val == value;
                case ">=":
-                  return val >= where.value;
+                  return val >= value;
                case ">":
-                  return val > where.value;
+                  return val > value;
                case "array-contains":
                   if (Array.isArray(val)) {
-                     return val.some(e => e === where.value);
+                     return val.some(e => e === value);
                   }
 
-                  break;
+                  return false;
                // case "array-contains-any":
-               // case "in":
+               case "in":
-
+                  if (typeof val === "object") {
+                     return value in val;
+                  }
+                  return false;
                default:
-                  throw new Error("Invalid where operation " + where.opStr);
+                  throw new QueryError("Invalid where operation " + opStr);
             }
          })
       }
@@ -409,9 +433,10 @@ export class CollectionQuery extends Query {
    async get() {
       let { collection, document } = await this.resolve(this.path);
       if (document)
-         throw new Error("Keys only works on collections!");
+         throw new QueryError("Keys only works on collections!");
       if (!collection)
-         throw new Error("There must be a collection");
+         return [];
+
       return new Promise<DocRes[]>((yes, no) => {
          const stream = this.database.data.iterator({
             ...this.getStreamOptions(collection),
@@ -440,13 +465,11 @@ export class CollectionQuery extends Query {
 
                   let data = decode(value);
                   if (this.fitsWhere(data)) {
-                     Logging.debug("Found fitting")
+                     if (this.limit < 0 || values.length < this.limit) {
-                     if (this.limit < 0 || value.length < this.limit) {
                         values.push({
                            id,
                            data
                         });
-
                      }
                      else {
                         stream.end((err) => err ? no(err) : yes(values))
@@ -459,7 +482,7 @@ export class CollectionQuery extends Query {
             }
          }
 
-         stream.next(onValue)
+         stream.next(onValue);
       })
    }
 
@@ -470,8 +493,8 @@ export class CollectionQuery extends Query {
 
    async snapshot(onChange: (change: (DocRes & { type: ChangeTypes })[]) => void) {
       if (this.subscription)
-         throw new Error("This query is already subscribed!");
+         throw new QueryError("This query is already subscribed!");
-      let { collection, document } = await this.resolve(this.path);
+      let { collection, document } = await this.resolve(this.path, true);
 
       let data = await this.get();
 
@@ -517,6 +540,9 @@ export class CollectionQuery extends Query {
 
 
    public async collections() {
+      if (!this.session.root)
+         throw new QueryError("No Permission!");
+
       return new Promise<string[]>((yes, no) => {
          let keys = [];
          const stream = this.database.data.createKeyStream({ keyAsBuffer: false })
@@ -527,10 +553,13 @@ export class CollectionQuery extends Query {
    }
 
    public async deleteCollection() {
+      if (!this.session.root)
+         throw new QueryError("No Permission!");
+
       const { collection, document, collectionKey } = await this.resolve(this.path);
 
       if (document) {
-         throw new Error("There can be no document defined on this operation");
+         throw new QueryError("There can be no document defined on this operation");
       }
 
       //TODO: Lock whole collection!
@@ -539,7 +568,8 @@ export class CollectionQuery extends Query {
       try {
          if (collection) {
             let documents = await this.keys();
-            for (let document in documents) {
+            // Logging.debug("To delete:", documents)
+            for (let document of documents) {
                batch.del(this.getKey(collection, document));
             }
             await batch.write();
@@ -556,3 +586,9 @@ export class CollectionQuery extends Query {
       return new CollectionQuery(...Query.getConstructorParams(query));
    }
 }
+
+export class QueryError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}

src/database/session.ts

@@ -1,8 +1,12 @@
+import { Query } from "./query";
+
 export default class Session {
    constructor(private _sessionid: string) { }
-   get sessionid() {
+   get id() {
       return this._sessionid;
    }
    root: boolean = false;
    uid: string = undefined;
+
+   queries = new Map<string, Query>();
 }

src/helper/jwt.ts

@@ -0,0 +1,12 @@
+import * as JWT from "jsonwebtoken";
+
+export async function verifyJWT(token: string, publicKey: string) {
+   return new Promise<any | undefined>((yes) => {
+      JWT.verify(token, publicKey, (err, decoded) => {
+         if (err)
+            yes(undefined);
+         else
+            yes(decoded);
+      })
+   })
+}

src/index.ts

@@ -5,9 +5,14 @@ import { DatabaseManager } from "./database/database";
 import { createServer } from "http";
 import { ConnectionManager } from "./connection";
 import { LoggingTypes } from "@hibas123/logging";
+import { readFileSync } from "fs";
 
 Logging.logLevel = config.dev ? LoggingTypes.Debug : LoggingTypes.Log;
 
+const version = JSON.parse(readFileSync("./package.json").toString()).version;
+
+Logging.log("Starting Database version:", version);
+
 DatabaseManager.init().then(() => {
    const http = createServer(Web.callback());
    ConnectionManager.bind(http);

src/web/helper/errors.ts

@@ -398,6 +398,12 @@ export class NoPermissionError extends HttpError {
    }
 }
 
+export class UnauthorizedError extends HttpError {
+   constructor(message: string) {
+      super(message, HttpStatusCode.UNAUTHORIZED)
+   }
+}
+
 export class BadRequestError extends HttpError {
    constructor(message: string) {
       super(message, HttpStatusCode.BAD_REQUEST)

src/web/helper/form.ts

@@ -1,4 +1,4 @@
-import getTemplate from "./hb";
+import { getTemplate } from "./hb";
 import { Context } from "vm";
 
 interface IFormConfigField {

src/web/helper/hb.ts

@@ -1,7 +1,7 @@
 import * as Handlebars from "handlebars";
 import { readFileSync } from "fs";
 import config from "../../config";
-import Logging from "@hibas123/logging";
+import Logging from "@hibas123/nodelogging";
 
 
 function checkCondition(v1, operator, v2) {
@@ -37,8 +37,22 @@ Handlebars.registerHelper('ifCond', function (v1, operator, v2, options) {
 
 
 const cache = new Map<string, Handlebars.TemplateDelegate>();
+const htmlCache = new Map<string, string>();
 
-export default function getTemplate(name: string) {
+export function getView(name: string) {
+   let tl: string;
+   if (!config.dev)
+      tl = htmlCache.get(name);
+
+   if (!tl) {
+      tl = readFileSync(`./views/${name}.html`).toString();
+      htmlCache.set(name, tl);
+   }
+
+   return tl;
+}
+
+export function getTemplate(name: string) {
    let tl: Handlebars.TemplateDelegate;
    if (!config.dev)
       tl = cache.get(name);

src/web/helper/table.ts

@@ -1,5 +1,5 @@
 import { Context } from "koa";
-import getTemplate from "./hb";
+import { getTemplate } from "./hb";
 
 export default function getTable(title: string, data: any[], ctx: Context) {
    let table: string[][] = [];

src/web/v1/admin.ts

@@ -6,7 +6,8 @@ import { BadRequestError, NoPermissionError } from "../helper/errors";
 import { DatabaseManager } from "../../database/database";
 import { MP } from "../../database/query";
 import config from "../../config";
-import Logging from "@hibas123/logging";
+import Logging from "@hibas123/nodelogging";
+import { getView } from "../helper/hb";
 
 const AdminRoute = new Router();
 
@@ -17,6 +18,11 @@ AdminRoute.use(async (ctx, next) => {
    return next();
 })
 
+AdminRoute.get("/", async ctx => {
+   //TODO: Main Interface
+   ctx.body = getView("admin");
+});
+
 AdminRoute.get("/settings", async ctx => {
    let res = await new Promise<string[][]>((yes, no) => {
       const stream = Settings.db.createReadStream({
@@ -51,11 +57,11 @@ AdminRoute.get("/data", async ctx => {
          keys: true,
          values: true,
          valueAsBuffer: true,
-         keyAsBuffer: false
+         keyAsBuffer: false,
+         limit: 1000
       });
       let res = [["key", "value"]];
       stream.on("data", ({ key, value }: { key: string, value: Buffer }) => {
-         Logging.debug("Admin Key:", key);
          res.push([key, key.split("/").length > 2 ? value.toString() : JSON.stringify(MP.decode(value))]);
       })
 
@@ -114,6 +120,48 @@ AdminRoute
       ctx.body = "Success";
    })
 
+AdminRoute.get("/collections", async ctx => {
+   const { database } = ctx.query;
+   let db = DatabaseManager.getDatabase(database);
+   if (!db)
+      throw new BadRequestError("Database not found");
+
+   let res = await new Promise<string[]>((yes, no) => {
+
+      const stream = db.collections.createKeyStream({
+         keyAsBuffer: false,
+         limit: 1000
+      });
+      let res = [];
+      stream.on("data", (key: string) => {
+         res.push(key);
+      })
+
+      stream.on("error", no);
+      stream.on("end", () => yes(res))
+   })
+
+   if (ctx.query.view) {
+      return getTable("Databases", res, ctx);
+   } else {
+      ctx.body = res;
+   }
+})
+
+AdminRoute.get("/collections/cleanup", async ctx => {
+   const { database } = ctx.query;
+   let db = DatabaseManager.getDatabase(database);
+   if (!db)
+      throw new BadRequestError("Database not found");
+
+   let deleted = await db.runCleanup();
+   if (ctx.query.view) {
+      return getTable("Databases", deleted, ctx);
+   } else {
+      ctx.body = deleted;
+   }
+})
+
 AdminRoute.get("/database/new", getForm("/v1/admin/database", "New/Change Database", {
    name: { label: "Name", type: "text", },
    accesskey: { label: "Access Key", type: "text" },

src/web/v1/index.ts

@@ -1,5 +1,59 @@
 import * as Router from "koa-router";
 import AdminRoute from "./admin";
+import { DatabaseManager } from "../../database/database";
+import { NotFoundError, NoPermissionError, BadRequestError } from "../helper/errors";
+import Logging from "@hibas123/nodelogging";
+import Session from "../../database/session";
+import nanoid = require("nanoid");
+import { verifyJWT } from "../../helper/jwt";
+import { QueryError } from "../../database/query";
 const V1 = new Router({ prefix: "/v1" });
+
 V1.use("/admin", AdminRoute.routes(), AdminRoute.allowedMethods());
+
+V1.post("/db/:database/query", async ctx => {
+   const { database } = ctx.params;
+   const { accesskey, authkey, rootkey } = ctx.query;
+
+   const query = ctx.request.body;
+   if (!query) {
+      throw new BadRequestError("Query not defined!");
+   }
+
+   const session = new Session(nanoid());
+   const db = DatabaseManager.getDatabase(database);
+   if (!db) {
+      throw new NotFoundError("Database not found!");
+   }
+
+   if (db.accesskey) {
+      if (!accesskey || accesskey !== db.accesskey) {
+         throw new NoPermissionError("");
+      }
+   }
+
+   if (authkey && db.publickey) {
+      let res = await verifyJWT(authkey, db.publickey);
+      if (!res || !res.uid) {
+         throw new BadRequestError("Invalid JWT");
+         return;
+      } else {
+         session.uid = res.uid;
+      }
+   }
+
+   if (rootkey && db.rootkey) {
+      if (rootkey === db.rootkey) {
+         session.root = true;
+         Logging.warning(`Somebody logged into ${database} via rootkey`);
+      }
+   }
+
+   ctx.body = await db.run(query, session).catch(err => {
+      if (err instanceof QueryError) {
+         throw new BadRequestError(err.message);
+      }
+      throw err;
+   })
+})
 export default V1;

views/admin.html

@@ -0,0 +1,112 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+   <meta charset="UTF-8">
+   <meta name="viewport" content="width=device-width, initial-scale=1.0">
+   <meta http-equiv="X-UA-Compatible" content="ie=edge">
+   <title>Admin Interface</title>
+   <link rel="stylesheet" href="https://unpkg.com/@hibas123/theme/out/base.css">
+   <link rel="stylesheet" href="https://unpkg.com/@hibas123/theme/out/light.css">
+
+   <script src="https://unpkg.com/handlebars/dist/handlebars.min.js"></script>
+
+   <style>
+      #message {
+         visibility: hidden;
+         background-color: lightgreen;
+         border: 1px solid lime;
+         border-radius: .5rem;
+         padding: 1rem;
+         font-size: 1.5rem;
+         margin-bottom: 1rem;
+      }
+
+      .grid {
+         display: grid;
+         height: 100vh;
+         grid-template-columns: 360px auto;
+      }
+
+      #content {
+         position: absolute;
+         top: 0;
+         left: 0;
+         width: 100%;
+         height: 100%;
+         border: 0;
+      }
+   </style>
+</head>
+
+<body>
+   <div class="grid">
+      <div style="border-right: 1px solid darkgrey; padding: 1rem;">
+         <h2>Navigation: </h2>
+         <ul class="list list-clickable">
+            <li onclick="loadView('settings');">Settings</li>
+            <li onclick="loadView('database', {full:true});">Databases</li>
+            <li onclick="loadView('database/new');">New Database</li>
+         </ul>
+         Databases:
+         <div id="dbs" class="list list-clickable" style="margin: 1rem;"></div>
+
+      </div>
+      <div style="position:relative;">
+         <iframe id="content"></iframe>
+      </div>
+   </div>
+
+   <template>
+
+   </template>
+
+   <script>
+      const key = new URL(window.location.href).searchParams.get("key");
+      const content = document.getElementById("content");
+      const base = new URL(window.location.href).host;
+
+      function getUrl(name, params, view = true) {
+         const url = new URL(window.location.href);
+         url.pathname = "/v1/admin/" + name;
+         for (let key in params || {})
+            url.searchParams.set(key, params[key]);
+
+         url.searchParams.set("key", key);
+         if (view)
+            url.searchParams.set("view", "true");
+
+         return url.href;
+      }
+
+      function loadView(name, params) {
+         content.src = getUrl(name, params);
+      }
+
+      loadView("settings")
+
+      const dbsul = document.getElementById("dbs");
+      function reloadDBs() {
+         fetch(getUrl("database", {}, false))
+            .then(res => res.json())
+            .then(databases => databases.map(database => `
+<div class="card margin elv-4">
+   <h3>${database}</h3>
+   <button class=btn onclick="loadView('data', {database:'${database}'})">Data</button>
+   <button class=btn onclick="loadView('collections', {database:'${database}'})">Collections</button>
+   <button class=btn onclick="loadView('collections/cleanup', {database:'${database}'})">Clean</button>
+</div>`
+            ))
+            .then(d => d.join("\n"))
+            .then(d => dbsul.innerHTML = d)
+            .catch(console.error)
+
+
+      }
+
+      reloadDBs();
+      setInterval(reloadDBs, 5000);
+   </script>
+</body>
+
+</html>