Adding batch support

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hibas123/realtimedb",
-  "version": "2.0.0-beta.8",
+  "version": "2.0.0-beta.9",
   "description": "",
   "main": "lib/index.js",
   "private": true,
@@ -18,29 +18,29 @@
   "devDependencies": {
     "@types/dotenv": "^8.2.0",
     "@types/jsonwebtoken": "^8.3.5",
-    "@types/koa": "^2.0.51",
+    "@types/koa": "^2.11.0",
     "@types/koa-router": "^7.0.42",
-    "@types/leveldown": "^4.0.1",
+    "@types/leveldown": "^4.0.2",
     "@types/levelup": "^3.1.1",
     "@types/nanoid": "^2.1.0",
-    "@types/node": "^12.12.5",
+    "@types/node": "^12.12.14",
-    "@types/ws": "^6.0.3",
+    "@types/ws": "^6.0.4",
     "concurrently": "^5.0.0",
-    "nodemon": "^1.19.4",
+    "nodemon": "^2.0.1",
-    "typescript": "^3.6.4"
+    "typescript": "^3.7.2"
   },
   "dependencies": {
-    "@hibas123/nodelogging": "^2.1.1",
+    "@hibas123/nodelogging": "^2.1.2",
-    "@hibas123/utils": "^2.1.1",
+    "@hibas123/utils": "^2.2.3",
     "dotenv": "^8.2.0",
-    "handlebars": "^4.5.1",
+    "handlebars": "^4.5.3",
     "jsonwebtoken": "^8.5.1",
     "koa": "^2.11.0",
     "koa-body": "^4.1.1",
     "koa-router": "^7.4.0",
     "leveldown": "^5.4.1",
     "levelup": "^4.3.2",
-    "nanoid": "^2.1.6",
+    "nanoid": "^2.1.7",
     "what-the-pack": "^2.0.3",
     "ws": "^7.2.0"
   }

src/connection.ts

@@ -1,8 +1,8 @@
 import Logging from "@hibas123/nodelogging";
 import { IncomingMessage, Server } from "http";
 import * as WebSocket from "ws";
-import { DatabaseManager, IQuery, ITypedQuery } from "./database/database";
+import { DatabaseManager } from "./database/database";
-import { CollectionQuery, DocumentQuery } from "./database/query";
+import { CollectionQuery, DocumentQuery, IQuery, ITypedQuery } from "./database/query";
 import Session from "./database/session";
 import { verifyJWT } from "./helper/jwt";
 import nanoid = require("nanoid");
@@ -61,14 +61,22 @@ export class ConnectionManager {
       }
 
       const answer = (id: string, data: any, error: boolean = false) => {
+         if (error)
+            Logging.error(error as any);
          socket.send(JSON.stringify({ ns: "message", data: { id, error, data } }));
       }
 
       const handler = new Map<string, ((data: any) => void)>();
 
-      handler.set("v2", async ({ id, query }: { id: string, query: IQuery }) => db.run(query, session)
+      handler.set("v2", async ({ id, query }) => db.run(Array.isArray(query) ? query : [query], session)
          .then(res => answer(id, res))
-         .catch(err => answer(id, undefined, err)));
+         .catch(err => answer(id, undefined, err))
+      );
+
+      // handler.set("bulk", async ({ id, query }) => db.run(query, session)
+      //    .then(res => answer(id, res))
+      //    .catch(err => answer(id, undefined, err))
+      // );
 
 
       const SnapshotMap = new Map<string, string>();
@@ -106,10 +114,8 @@ export class ConnectionManager {
 
       socket.on("close", () => {
          Logging.log(`${session.id} has disconnected!`);
-         session.queries.forEach((query: DocumentQuery | CollectionQuery) => {
+         session.subscriptions.forEach(unsubscribe => unsubscribe());
-            query.unsubscribe();
+         session.subscriptions.clear();
-         })
-         session.queries.clear();
          socket.removeAllListeners();
       })
    }

src/database/database.ts

@@ -1,29 +1,18 @@
 import { Rules } from "./rules";
 import Settings from "../settings";
-import getLevelDB, { LevelDB, deleteLevelDB } from "../storage";
+import getLevelDB, { LevelDB, deleteLevelDB, resNull } from "../storage";
 import DocumentLock from "./lock";
-import { DocumentQuery, CollectionQuery, Query, QueryError } from "./query";
+import { DocumentQuery, CollectionQuery, Query, QueryError, ITypedQuery, IQuery } from "./query";
 import Logging from "@hibas123/nodelogging";
 import Session from "./session";
-import nanoid = require("nanoid");
+import nanoid = require("nanoid/generate");
+import { Observable } from "@hibas123/utils";
 
-type IWriteQueries = "set" | "update" | "delete" | "add";
+const ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
-type ICollectionQueries = "get" | "add" | "keys" | "delete-collection" | "list";
-type IDocumentQueries = "get" | "set" | "update" | "delete";
-
-export interface ITypedQuery<T> {
-   path: string[];
-   type: T;
-   data?: any;
-   options?: any;
-}
-
-interface ITransaction {
-   queries: ITypedQuery<IWriteQueries>[];
-}
-
-export type IQuery = ITypedQuery<ICollectionQueries | IDocumentQueries>;
 
+// interface ITransaction {
+//    queries: ITypedQuery<IWriteQueries>[];
+// }
 
 export class DatabaseManager {
    static databases = new Map<string, Database>();
@@ -66,12 +55,17 @@ export type ChangeTypes = "added" | "modified" | "deleted";
 export type Change = {
    data: any;
    document: string;
+   collection: string;
    type: ChangeTypes;
    sender: string;
 }
 
 
 export class Database {
+   public static getKey(collectionid: string, documentid?: string) {
+      return `${collectionid || ""}/${documentid || ""}`;
+   }
+
    private level = getLevelDB(this.name);
 
    get data() {
@@ -84,10 +78,15 @@ export class Database {
 
 
    public rules: Rules;
-   public locks = new DocumentLock()
+   private locks = new DocumentLock()
    public collectionLocks = new DocumentLock()
 
-   public changes = new Map<string, Set<(change: Change) => void>>();
+   public changeListener = new Map<string, Set<(change: Change[]) => void>>();
+   public collectionChangeListener = new Observable<{
+      key: string;
+      id: string;
+      type: "create" | "delete"
+   }>();
 
    toJSON() {
       return {
@@ -124,14 +123,71 @@ export class Database {
       this.publickey = key;
    }
 
+   public async resolve(path: string[], create = false): Promise<{ collection: string, document: string, collectionKey: string }> {
+      path = [...path]; // Create modifiable copy
+      let collectionID: string = undefined;
+      let documentKey = path.length % 2 === 0 ? path.pop() : undefined;
+      let key = path.join("/");
 
-   getQuery(path: string[], session: Session, type: "document" | "collection" | "any") {
+      const lock = await this.collectionLocks.lock(key);
-      if (type === "document")
+
-         return new DocumentQuery(this, path, session);
+      try {
-      else if (type === "collection")
+         collectionID = await this.collections.get(key).then(r => r.toString()).catch(resNull);
-         return new CollectionQuery(this, path, session);
+         if (!collectionID && create) {
-      else
+            collectionID = nanoid(ALPHABET, 32);
-         return new Query(this, path, session);
+            await this.collections.put(key, collectionID);
+            setImmediate(() => {
+               this.collectionChangeListener.send({
+                  id: collectionID,
+                  key,
+                  type: "create"
+               })
+            })
+         }
+      } finally {
+         lock();
+      }
+
+      return {
+         collection: collectionID,
+         document: documentKey,
+         collectionKey: key
+      };
+   }
+
+   private sendChanges(changes: Change[]) {
+      let col = new Map<string, Map<string, Change[]>>();
+      changes.forEach(change => {
+         let e = col.get(change.collection);
+         if (!e) {
+            e = new Map()
+            col.set(change.collection, e);
+         }
+
+         let d = e.get(change.document);
+         if (!d) {
+            d = [];
+            e.set(change.document, d);
+         }
+
+         d.push(change);
+      })
+
+      setImmediate(() => {
+         for (let [collection, documents] of col.entries()) {
+            let collectionChanges = [];
+            for (let [document, documentChanges] of documents.entries()) {
+               let s = this.changeListener.get(Database.getKey(collection, document));
+               if (s)
+                  s.forEach(e => setImmediate(() => e(documentChanges)));
+
+               collectionChanges.push(...documentChanges);
+            }
+            let s = this.changeListener.get(Database.getKey(collection))
+            if (s)
+               s.forEach(e => setImmediate(() => e(collectionChanges)))
+         }
+      })
    }
 
    private validate(query: ITypedQuery<any>) {
@@ -146,80 +202,121 @@ export class Database {
          throw inv;
    }
 
-   async run(query: IQuery, session: Session) {
+   async run(queries: IQuery[], session: Session) {
-      this.validate(query);
+      let resolve: { path: string[], create: boolean, resolved?: [string, string, string] }[] = [];
-      const isCollection = query.path.length % 2 === 1;
+
-      if (isCollection) {
+      const addToResolve = (path: string[], create?: boolean) => {
-         const q = new CollectionQuery(this, query.path, session);
+         let entry = resolve.find(e => { //TODO: Find may be slow...
-         let type = query.type as ICollectionQueries;
+            if (e.path.length !== path.length)
-         switch (type) {
+               return false;
-            case "add":
+            for (let i = 0; i < e.path.length; i++) {
-               return q.add(query.data);
+               if (e.path[i] !== path[i])
-            case "get":
+                  return false;
-               const limit = (query.options || {}).limit;
+            }
-               if (limit)
+            return true;
-                  q.limit = limit;
+         })
-               const where = (query.options || {}).where;
+
-               if (where)
+         if (!entry) {
-                  q.where = where;
+            entry = {
-               return q.get();
+               path,
-            case "keys":
+               create
-               return q.keys();
+            }
-            case "list":
+            resolve.push(entry);
-               return q.collections();
-            case "delete-collection":
-               return q.deleteCollection();
-            default:
-               return Promise.reject(new Error("Invalid query!"));
-         }
-      } else {
-         const q = new DocumentQuery(this, query.path, session);
-         let type = query.type as IDocumentQueries;
-         switch (type) {
-            case "get":
-               return q.get();
-            case "set":
-               return q.set(query.data, query.options || {});
-            case "update":
-               return q.update(query.data);
-            case "delete":
-               return q.delete();
-            default:
-               return Promise.reject(new Error("Invalid query!"));
          }
+
+         entry.create = entry.create || create;
+
+         return entry;
       }
+
+      const isBatch = queries.length > 1;
+      let parsed = queries.map(rawQuery => {
+         this.validate(rawQuery);
+         const isCollection = rawQuery.path.length % 2 === 1;
+
+         let query = isCollection
+            ? new CollectionQuery(this, session, rawQuery)
+            : new DocumentQuery(this, session, rawQuery);
+
+         if (isBatch && !query.batchCompatible)
+            throw new Error("There are queries that are not batch compatible!");
+
+         let path = addToResolve(rawQuery.path, query.createCollection);
+         if (query.additionalLock)
+            addToResolve(query.additionalLock);
+
+         return {
+            path,
+            query
+         };
+      });
+
+      resolve = resolve.sort((a, b) => a.path.length - b.path.length);
+
+      let locks: (() => void)[] = [];
+      for (let e of resolve) {
+         let { collection, document, collectionKey } = await this.resolve(e.path, e.create);
+         e.resolved = [collection, document, collectionKey];
+
+         locks.push(
+            await this.locks.lock(collection, document)
+         );
+      }
+
+      let result = [];
+      try {
+         let batch = this.data.batch();
+         let changes: Change[] = [];
+         for (let e of parsed) {
+            result.push(
+               await e.query.run(e.path.resolved[0], e.path.resolved[1], batch, e.path.resolved[2])
+            );
+            changes.push(...e.query.changes);
+         }
+         if (batch.length > 0)
+            await batch.write();
+
+         this.sendChanges(changes);
+      } finally {
+         locks.forEach(lock => lock());
+      }
+
+      if (isBatch)
+         return result;
+      else
+         return result[0]
    }
 
-   async snapshot(query: ITypedQuery<"snapshot">, session: Session, onchange: (change: any) => void) {
+   async snapshot(rawQuery: ITypedQuery<"snapshot">, session: Session, onchange: (change: any) => void) {
-      this.validate(query);
+      Logging.debug("Snaphot request:", rawQuery.path);
+      this.validate(rawQuery);
 
-      const isCollection = query.path.length % 2 === 1;
+      if (rawQuery.type !== "snapshot")
-      let q: DocumentQuery | CollectionQuery;
+         throw new Error("Invalid query type!");
-      if (isCollection) {
-         q = new CollectionQuery(this, query.path, session);
-         const limit = (query.options || {}).limit;
-         if (limit)
-            q.limit = limit;
-         const where = (query.options || {}).where;
-         if (where)
-            q.where = where;
-      } else {
-         q = new DocumentQuery(this, query.path, session);
-      }
 
-      const id = nanoid(16);
+      const isCollection = rawQuery.path.length % 2 === 1;
-      session.queries.set(id, q);
+      let query = isCollection
+         ? new CollectionQuery(this, session, rawQuery, true)
+         : new DocumentQuery(this, session, rawQuery, true);
+
+      const {
+         unsubscribe,
+         value
+      } = await query.snapshot(onchange);
+
+      const id = nanoid(ALPHABET, 16);
+      session.subscriptions.set(id, unsubscribe);
       return {
          id,
-         snaphot: await q.snapshot(onchange)
+         snaphot: value
       };
    }
 
    async unsubscribe(id: string, session: Session) {
-      let query: CollectionQuery | DocumentQuery = session.queries.get(id) as any;
+      let query = session.subscriptions.get(id);
       if (query) {
-         query.unsubscribe();
+         query();
-         session.queries.delete(id);
+         session.subscriptions.delete(id);
       }
    }
 

src/database/lock.ts

@@ -8,6 +8,7 @@ export default class DocumentLock {
    }
 
    async lock(collection: string = "", document: string = "") {
+      //TODO: Check collection locks
       let key = collection + "/" + document;
       let l = this.locks.get(key);
       if (l)

src/database/query.ts

@@ -4,6 +4,20 @@ import nanoid = require("nanoid/generate");
 import Logging from "@hibas123/nodelogging";
 import * as MSGPack from "what-the-pack";
 import Session from "./session";
+import { LevelUpChain } from "levelup";
+
+export type IWriteQueries = "set" | "update" | "delete" | "add";
+export type ICollectionQueries = "get" | "add" | "keys" | "delete-collection" | "list";
+export type IDocumentQueries = "get" | "set" | "update" | "delete";
+
+export interface ITypedQuery<T> {
+   path: string[];
+   type: T;
+   data?: any;
+   options?: any;
+}
+
+export type IQuery = ITypedQuery<ICollectionQueries | IDocumentQueries | "snapshot">;
 
 export const MP = MSGPack.initialize(2 ** 20);
 
@@ -11,7 +25,22 @@ const ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
 
 const { encode, decode } = MP;
 
-export class Query {
+type Runner = (collection: string, document: string, batch: LevelUpChain, collectionKey: string) => any;
+
+interface IPreparedQuery {
+   createCollection: boolean;
+   needDocument: boolean;
+   batchCompatible: boolean;
+   runner: Runner;
+   additionalLock?: string[];
+}
+
+interface DocRes {
+   id: string;
+   data: any;
+}
+
+export abstract class Query {
    /**
     * Returns true if the path only contains valid characters and false if it doesn't
     * @param path Path to be checked
@@ -20,48 +49,37 @@ export class Query {
       return path.every(e => (e.match(/[^a-zA-Z0-9_\-\<\>]/g) || []).length === 0);
    }
 
-   constructor(protected database: Database, protected path: string[], protected session: Session) {
+   public changes: Change[] = [];
-      if (path.length > 10) {
+
+   public readonly createCollection: boolean;
+   public readonly needDocument: boolean;
+   public readonly batchCompatible: boolean;
+   public readonly additionalLock?: string[];
+   private readonly _runner: Runner;
+
+   constructor(protected database: Database, protected session: Session, protected query: IQuery, snapshot = false) {
+      if (query.path.length > 10) {
          throw new QueryError("Path is to long. Path is only allowed to be 10 Layers deep!");
       }
-      if (!this.validatePath(path)) {
+      if (!this.validatePath(query.path)) {
          throw new QueryError("Path can only contain a-z A-Z 0-9  '-' '-' '<' and '>' ");
       }
-   }
 
-
+      if (!snapshot) {
-   protected async resolve(path: string[], create = false): Promise<{ collection: string, document: string, collectionKey: string }> {
+         let data = this.prepare(query);
-      path = [...path]; // Create modifiable copy
+         this.createCollection = data.createCollection;
-      let collectionID: string = undefined;
+         this.needDocument = data.needDocument;
-      let documentKey = path.length % 2 === 0 ? path.pop() : undefined;
+         this.batchCompatible = data.batchCompatible;
-      let key = path.join("/");
+         this.additionalLock = data.additionalLock;
-
+         this._runner = data.runner;
-      const lock = await this.database.collectionLocks.lock(key);
-
-      try {
-         collectionID = await this.database.collections.get(key).then(r => r.toString()).catch(resNull);
-         if (!collectionID && create) {
-            collectionID = nanoid(ALPHABET, 32);
-            await this.database.collections.put(key, collectionID);
-         }
-      } finally {
-         lock();
       }
-
-      return {
-         collection: collectionID,
-         document: documentKey,
-         collectionKey: key
-      };
    }
 
-   protected getKey(collection: string, document?: string) {
+   protected abstract prepare(query: IQuery): IPreparedQuery;
-      return `${collection || ""}/${document || ""}`;
-   }
 
    protected getDoc(collection: string, document: string) {
       return this.database.data
-         .get(this.getKey(collection, document), { asBuffer: true })
+         .get(Database.getKey(collection, document), { asBuffer: true })
          .then(res => decode<any>(res as Buffer)).catch(resNull);
    }
 
@@ -69,22 +87,90 @@ export class Query {
       let change: Change = {
          type,
          document,
+         collection,
          data,
          sender: this.session.id
       }
 
-      let s = this.database.changes.get(this.getKey(collection, document))
+      this.changes.push(change);
-
-      if (s)
-         s.forEach(e => setImmediate(() => e(change)))
-      s = this.database.changes.get(this.getKey(collection))
-      if (s)
-         s.forEach(e => setImmediate(() => e(change)))
-
    }
 
-   protected static getConstructorParams(query: Query): [Database, string[], Session] {
+   protected static getConstructorParams(query: Query): [Database, Session, IQuery] {
-      return [query.database, query.path, query.session];
+      return [query.database, query.session, query.query];
+   }
+
+   protected abstract checkChange(change: Change): boolean;
+   protected abstract firstSend(collection: string, document: string): Promise<any>;
+
+   public run(collection: string, document: string, batch: LevelUpChain, collectionKey: string) {
+      return this._runner.call(this, collection, document, batch, collectionKey);
+   }
+
+   public async snapshot(onChange: (change: (DocRes & { type: ChangeTypes })[]) => void) {
+      const receivedChanges = (changes: Change[]) => {
+         let res = changes.filter(change => this.checkChange(change)).map(change => {
+            return {
+               id: change.document,
+               data: change.data,
+               type: change.type
+            }
+         })
+         if (res.length > 0)
+            onChange(res);
+      };
+
+      const unsub = this.database.collectionChangeListener.subscribe(change => {
+         if (change.key === collectionKey) {
+            if (change.type === "create")
+               addSubscriber(change.id);
+            else
+               removeSubscriber(); // Send delete for all elements (Don't know how to do this...)
+         }
+      })
+
+
+      let { collection, document, collectionKey } = await this.database.resolve(this.query.path)
+      let oldKey: string = undefined;
+
+      const removeSubscriber = () => {
+         if (!oldKey)
+            return;
+         let s = this.database.changeListener.get(oldKey);
+         if (s) {
+            s.delete(receivedChanges);
+            if (s.size <= 0)
+               this.database.changeListener.delete(oldKey);
+         }
+         oldKey = undefined;
+      }
+
+      const addSubscriber = (collection: string) => {
+         let key = Database.getKey(collection, document);
+         if (oldKey !== key) {
+            if (oldKey !== undefined)
+               removeSubscriber();
+
+            let s = this.database.changeListener.get(key);
+            if (!s) {
+               s = new Set();
+               this.database.changeListener.set(key, s);
+            }
+
+            s.add(receivedChanges);
+         }
+      }
+
+      if (collection) {
+         addSubscriber(collection);
+      }
+
+      return {
+         unsubscribe: () => {
+            unsub();
+            removeSubscriber();
+         },
+         value: await this.firstSend(collection, document)
+      }
    }
 }
 
@@ -95,185 +181,140 @@ interface UpdateData {
    }
 }
 export class DocumentQuery extends Query {
-   constructor(database: Database, path: string[], session: Session) {
+   prepare(query: IQuery): IPreparedQuery {
-      super(database, path, session);
+      let type = query.type as IDocumentQueries;
-      this.onChange = this.onChange.bind(this);
+      switch (type) {
+         case "get":
+            return {
+               batchCompatible: false,
+               createCollection: false,
+               needDocument: false,
+               runner: this.get
+            }
+         case "set":
+            return {
+               batchCompatible: true,
+               createCollection: true,
+               needDocument: true,
+               runner: this.set
+            }
+         case "update":
+            return {
+               batchCompatible: true,
+               createCollection: true,
+               needDocument: true,
+               runner: this.update
+            }
+         case "delete":
+            return {
+               batchCompatible: true,
+               createCollection: false,
+               needDocument: true,
+               runner: this.delete
+            }
+         default:
+            throw new Error("Invalid query type: " + type);
+      }
    }
 
-   public async get() {
+   private async get(collection: string, document: string) {
-      let { collection, document } = await this.resolve(this.path);
-
       if (!collection || !document) {
          return null;
       }
 
-      return this.getDoc(collection, document)
+      return this.getDoc(collection, document);
    }
 
-   public async set(data: any, { merge = false }) {
+   private async set(collection: string, document: string, batch?: LevelUpChain) {
+      const { data, options } = this.query;
       if (data === null)
-         return this.delete();
+         return this.delete(collection, document, batch);
-      let { collection, document } = await this.resolve(this.path, true);
-      if (!collection) {
-         throw new QueryError("There must be a collection!")
-      }
 
-      if (!document) {
-         throw new QueryError("There must be a document key!")
-      }
-
-      const lock = await this.database.locks.lock(collection, document);
 
       let isNew = !(await this.getDoc(collection, document))
-
+      batch.put(Database.getKey(collection, document), encode(data))
-      return this.database.data
+      this.sendChange(collection, document, isNew ? "added" : "modified", data)
-         .put(this.getKey(collection, document), encode(data))
-         .then(() => this.sendChange(collection, document, isNew ? "added" : "modified", data))
-         .finally(() => lock())
    }
 
-   public async update(updateData: UpdateData) {
+   private async update(collection: string, document: string, batch?: LevelUpChain) {
-      let { collection, document } = await this.resolve(this.path, true);
+      const updateData: UpdateData = this.query.data;
-      if (!collection) {
-         throw new QueryError("There must be a collection!")
-      }
-
-      if (!document) {
-         throw new QueryError("There must be a document key!")
-      }
-
-      // Logging.debug(updateData);
-
-      const lock = await this.database.locks.lock(collection, document);
-      try {
-         let data = await this.getDoc(collection, document);
-         let isNew = false
-         if (!data) {
-            isNew = true;
-            data = {};
-         }
-
-         for (let path in updateData) {
-            const toUpdate = updateData[path];
-            let d = data;
-            let parts = path.split(".");
-            while (parts.length > 1) {
-               let seg = parts.shift();
-               if (!data[seg])
-                  data[seg] = {}
-               d = data[seg];
-            }
-
-            const last = parts[0];
-
-            // Logging.debug(parts, last, d)
-
-            switch (toUpdate.type) {
-               case "value":
-                  d[last] = toUpdate.value;
-                  break;
-               case "increment":
-                  if (d[last] === undefined || d[last] === null)
-                     d[last] = toUpdate.value;
-                  else if (typeof d[last] !== "number") {
-                     throw new QueryError("Field is no number!");
-                  } else {
-                     d[last] += toUpdate.value;
-                  }
-                  break;
-               case "timestamp":
-                  d[last] = new Date().valueOf();
-                  break;
-               case "push":
-                  if (d[last] === undefined || d[last] === null)
-                     d[last] = [toUpdate.value];
-                  else if (Array.isArray(d[last])) {
-                     d[last].push(toUpdate.value);
-                  } else {
-                     throw new QueryError("Field is not array!");
-                  }
-                  break;
-               default:
-                  throw new QueryError("Invalid update type: " + toUpdate.type);
-            }
-         }
-
-         this.database.data
-            .put(this.getKey(collection, document), encode(data))
-            .then(() => this.sendChange(collection, document, isNew ? "added" : "modified", data))
-      } finally {
-         lock();
-      }
-      //TODO: Implement
-   }
-
-   public async delete() {
-      let { collection, document } = await this.resolve(this.path);
-
-      if (!collection) {
-         throw new QueryError("There must be a collection!")
-      }
-
-      if (!document) {
-         throw new QueryError("There must be a document key!")
-      }
-
-      const lock = await this.database.locks.lock(collection, document);
-
-      return await this.database.data
-         .del(`${collection}/${document}`)
-         .then(() => this.sendChange(collection, document, "deleted", null))
-         .finally(() => lock())
-   }
-
-
-
-   private subscription: {
-      key: string,
-      onChange: (change: DocRes & { type: ChangeTypes }) => void
-   };
-
-   async snapshot(onChange: (change: DocRes & { type: ChangeTypes }) => void) {
-      if (this.subscription)
-         throw new QueryError("This query is already subscribed!");
-      let { collection, document } = await this.resolve(this.path);
 
       let data = await this.getDoc(collection, document);
-      let key = this.getKey(collection, document);
+      let isNew = false
-      this.subscription = {
+      if (!data) {
-         key,
+         isNew = true;
-         onChange
+         data = {};
-      }
-      let s = this.database.changes.get(key);
-      if (!s) {
-         s = new Set();
-         this.database.changes.set(key, s);
       }
 
-      s.add(this.onChange);
+      for (let path in updateData) {
+         const toUpdate = updateData[path];
+         let d = data;
+         let parts = path.split(".");
+         while (parts.length > 1) {
+            let seg = parts.shift();
+            if (!data[seg])
+               data[seg] = {}
+            d = data[seg];
+         }
 
-      return data;
+         const last = parts[0];
+
+         switch (toUpdate.type) {
+            case "value":
+               d[last] = toUpdate.value;
+               break;
+            case "increment":
+               if (d[last] === undefined || d[last] === null)
+                  d[last] = toUpdate.value;
+               else if (typeof d[last] !== "number") {
+                  throw new QueryError("Field is no number!");
+               } else {
+                  d[last] += toUpdate.value;
+               }
+               break;
+            case "timestamp":
+               d[last] = new Date().valueOf();
+               break;
+            case "push":
+               if (d[last] === undefined || d[last] === null)
+                  d[last] = [toUpdate.value];
+               else if (Array.isArray(d[last])) {
+                  d[last].push(toUpdate.value);
+               } else {
+                  throw new QueryError("Field is not array!");
+               }
+               break;
+            default:
+               throw new QueryError("Invalid update type: " + toUpdate.type);
+         }
+      }
+
+      if (batch) {
+         batch.put(Database.getKey(collection, document), encode(data))
+      } else {
+         await this.database.data
+            .put(Database.getKey(collection, document), encode(data))
+      }
+
+      this.sendChange(collection, document, isNew ? "added" : "modified", data)
    }
 
-   onChange(change: Change) {
+   private async delete(collection: string, document: string, batch?: LevelUpChain) {
-      // if(change.sender === this.sender)
+      if (batch) {
-      //    return
+         batch.del(Database.getKey(collection, document))
-      this.subscription.onChange({
+      } else {
-         id: change.document,
+         await this.database.data.del(Database.getKey(collection, document));
-         data: change.data,
+      }
-         type: change.type
+
-      })
+      this.sendChange(collection, document, "deleted", null)
    }
 
-   unsubscribe() {
+   checkChange(change: Change) {
-      if (!this.subscription)
+      return true;
-         return;
+   }
-      let s = this.database.changes.get(this.subscription.key);
-      s.delete(this.onChange);
-      if (s.size <= 0)
-         this.database.changes.delete(this.subscription.key);
 
-      this.subscription = undefined;
+   firstSend(collection: string, document: string) {
+      return this.get(collection, document);
    }
 
    public static fromQuery(query: Query) {
@@ -302,15 +343,61 @@ type IQueryWhereArray = [FieldPath, WhereFilterOp, any];
 
 type IQueryWhere = IQueryWhereArray | IQueryWhereVerbose;
 
-interface DocRes {
-   id: string;
-   data: any;
-}
-
 export class CollectionQuery extends Query {
-   constructor(database: Database, path: string[], session: Session) {
+   private _addId: string;
-      super(database, path, session);
+
-      this.onChange = this.onChange.bind(this);
+
+   prepare(query): IPreparedQuery {
+      switch (query.type as ICollectionQueries) {
+         case "add":
+            this._addId = nanoid(ALPHABET, 32)
+            return {
+               batchCompatible: true,
+               createCollection: true,
+               needDocument: false,
+               runner: this.add,
+               additionalLock: [...query.path, this._addId]
+            }
+         case "get":
+            const limit = (query.options || {}).limit;
+            if (limit)
+               this.limit = limit;
+            const where = (query.options || {}).where;
+            if (where)
+               this.where = where;
+
+            return {
+               batchCompatible: false,
+               createCollection: false,
+               needDocument: false,
+               runner: this.get
+            }
+         case "keys":
+            return {
+               batchCompatible: false,
+               createCollection: false,
+               needDocument: false,
+               runner: this.keys
+            }
+         case "list":
+            return {
+               batchCompatible: false,
+               createCollection: false,
+               needDocument: false,
+               runner: this.keys
+            }
+         case "delete-collection":
+            return {
+               batchCompatible: false,
+               createCollection: false,
+               needDocument: false,
+               runner: this.deleteCollection
+            }
+         // run = () => q.deleteCollection();
+         // break;
+         default:
+            throw new Error("Invalid query!");
+      }
    }
 
 
@@ -338,15 +425,19 @@ export class CollectionQuery extends Query {
 
    public limit: number = -1;
 
-   public async add(value: any) {
+   public async add(collection: string, document: string, batch: LevelUpChain, collectionKey: string) {
-      let id = nanoid(ALPHABET, 32);
+      let q = new DocumentQuery(this.database, this.session, {
-      let q = new DocumentQuery(this.database, [...this.path, id], this.session);
+         type: "set",
-      await q.set(value, {});
+         path: this.additionalLock,
-      return id;
+         data: this.query.data,
+         options: this.query.options
+      });
+      await q.run(collection, this._addId, batch, collectionKey);
+      return this._addId;
    }
 
    private getStreamOptions(collection: string) {
-      let gt = Buffer.from(this.getKey(collection) + " ");
+      let gt = Buffer.from(Database.getKey(collection) + " ");
       gt[gt.length - 1] = 0;
 
       let lt = Buffer.alloc(gt.length);
@@ -359,10 +450,7 @@ export class CollectionQuery extends Query {
       }
    }
 
-   public async keys() {
+   public async keys(collection: string) {
-      let { collection, document } = await this.resolve(this.path);
-      if (document)
-         throw new QueryError("Keys only works on collections!");
       if (!collection)
          return []
 
@@ -382,7 +470,7 @@ export class CollectionQuery extends Query {
       });
    }
 
-   private getFieldValue(data: any, path: FieldPath) {
+   private _getFieldValue(data: any, path: FieldPath) {
       let parts = path.split(".");
       let d = data;
       while (parts.length > 0) {
@@ -395,10 +483,10 @@ export class CollectionQuery extends Query {
       return d;
    }
 
-   private fitsWhere(data: any): boolean {
+   private _fitsWhere(data: any): boolean {
       if (this._where.length > 0) {
          return this._where.every(([fieldPath, opStr, value]) => {
-            let val = this.getFieldValue(data, fieldPath);
+            let val = this._getFieldValue(data, fieldPath);
             switch (opStr) {
                case "<":
                   return val < value;
@@ -430,10 +518,7 @@ export class CollectionQuery extends Query {
       return true;
    }
 
-   async get() {
+   async get(collection: string) {
-      let { collection, document } = await this.resolve(this.path);
-      if (document)
-         throw new QueryError("Keys only works on collections!");
       if (!collection)
          return [];
 
@@ -464,7 +549,7 @@ export class CollectionQuery extends Query {
                   const id = s[1];
 
                   let data = decode(value);
-                  if (this.fitsWhere(data)) {
+                  if (this._fitsWhere(data)) {
                      if (this.limit < 0 || values.length < this.limit) {
                         values.push({
                            id,
@@ -486,59 +571,14 @@ export class CollectionQuery extends Query {
       })
    }
 
-   private subscription: {
+   checkChange(change: Change) {
-      key: string,
+      return this._fitsWhere(change.data);
-      onChange: (change: (DocRes & { type: ChangeTypes })[]) => void
-   };
-
-   async snapshot(onChange: (change: (DocRes & { type: ChangeTypes })[]) => void) {
-      if (this.subscription)
-         throw new QueryError("This query is already subscribed!");
-      let { collection, document } = await this.resolve(this.path, true);
-
-      let data = await this.get();
-
-      let key = this.getKey(collection, document);
-      this.subscription = {
-         key,
-         onChange
-      }
-      let s = this.database.changes.get(key);
-      if (!s) {
-         s = new Set();
-         this.database.changes.set(key, s);
-      }
-
-      s.add(this.onChange);
-
-      return data;
    }
 
-   onChange(change: Change) {
+   firstSend(collection: string) {
-      // if(change.sender === this.sender)
+      return this.get(collection)
-      //    return
-
-      if (this.fitsWhere(change.data)) {
-         this.subscription.onChange([{
-            id: change.document,
-            data: change.data,
-            type: change.type
-         }])
-      }
    }
 
-   unsubscribe() {
-      if (!this.subscription)
-         return;
-      let s = this.database.changes.get(this.subscription.key);
-      s.delete(this.onChange);
-      if (s.size <= 0)
-         this.database.changes.delete(this.subscription.key);
-
-      this.subscription = undefined;
-   }
-
-
    public async collections() {
       if (!this.session.root)
          throw new QueryError("No Permission!");
@@ -552,29 +592,27 @@ export class CollectionQuery extends Query {
       });
    }
 
-   public async deleteCollection() {
+   public async deleteCollection(collection: string, document: string, _b: LevelUpChain, collectionKey: string) {
       if (!this.session.root)
          throw new QueryError("No Permission!");
 
-      const { collection, document, collectionKey } = await this.resolve(this.path);
-
-      if (document) {
-         throw new QueryError("There can be no document defined on this operation");
-      }
-
       //TODO: Lock whole collection!
-
       let batch = this.database.data.batch();
       try {
          if (collection) {
-            let documents = await this.keys();
+            let documents = await this.keys(collection);
             // Logging.debug("To delete:", documents)
             for (let document of documents) {
-               batch.del(this.getKey(collection, document));
+               batch.del(Database.getKey(collection, document));
             }
             await batch.write();
             batch = undefined;
             await this.database.collections.del(collectionKey);
+            this.database.collectionChangeListener.send({
+               id: collection,
+               key: collectionKey,
+               type: "delete"
+            });
          }
       } finally {
          if (batch)

src/database/session.ts

@@ -1,4 +1,3 @@
-import { Query } from "./query";
 
 export default class Session {
    constructor(private _sessionid: string) { }
@@ -8,5 +7,5 @@ export default class Session {
    root: boolean = false;
    uid: string = undefined;
 
-   queries = new Map<string, Query>();
+   subscriptions = new Map<string, (() => void)>();
 }

src/web/v1/index.ts

@@ -49,7 +49,7 @@ V1.post("/db/:database/query", async ctx => {
       }
    }
 
-   ctx.body = await db.run(query, session).catch(err => {
+   ctx.body = await db.run([query], session).catch(err => {
       if (err instanceof QueryError) {
          throw new BadRequestError(err.message);
       }