OpenServer/RealtimeDB-OLD
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Adding batch support

Browse Source
This commit is contained in:
Fabian Stamm 2019-12-01 03:34:25 +01:00
parent 2ac9def153
commit 904b986e22
8 changed files with 637 additions and 2130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1840
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

22
package.json
View File

@ -1,6 +1,6 @@
{
"name": "@hibas123/realtimedb",
"version": "2.0.0-beta.8",
"version": "2.0.0-beta.9",
"description": "",
"main": "lib/index.js",
"private": true,
@ -18,29 +18,29 @@
"devDependencies": {
"@types/dotenv": "^8.2.0",
"@types/jsonwebtoken": "^8.3.5",
"@types/koa": "^2.0.51",
"@types/koa": "^2.11.0",
"@types/koa-router": "^7.0.42",
"@types/leveldown": "^4.0.1",
"@types/leveldown": "^4.0.2",
"@types/levelup": "^3.1.1",
"@types/nanoid": "^2.1.0",
"@types/node": "^12.12.5",
"@types/ws": "^6.0.3",
"@types/node": "^12.12.14",
"@types/ws": "^6.0.4",
"concurrently": "^5.0.0",
"nodemon": "^1.19.4",
"typescript": "^3.6.4"
"nodemon": "^2.0.1",
"typescript": "^3.7.2"
},
"dependencies": {
"@hibas123/nodelogging": "^2.1.1",
"@hibas123/utils": "^2.1.1",
"@hibas123/nodelogging": "^2.1.2",
"@hibas123/utils": "^2.2.3",
"dotenv": "^8.2.0",
"handlebars": "^4.5.1",
"handlebars": "^4.5.3",
"jsonwebtoken": "^8.5.1",
"koa": "^2.11.0",
"koa-body": "^4.1.1",
"koa-router": "^7.4.0",
"leveldown": "^5.4.1",
"levelup": "^4.3.2",
"nanoid": "^2.1.6",
"nanoid": "^2.1.7",
"what-the-pack": "^2.0.3",
"ws": "^7.2.0"
}

22
src/connection.ts
View File

@ -1,8 +1,8 @@
import Logging from "@hibas123/nodelogging";
import { IncomingMessage, Server } from "http";
import * as WebSocket from "ws";
import { DatabaseManager, IQuery, ITypedQuery } from "./database/database";
import { CollectionQuery, DocumentQuery } from "./database/query";
import { DatabaseManager } from "./database/database";
import { CollectionQuery, DocumentQuery, IQuery, ITypedQuery } from "./database/query";
import Session from "./database/session";
import { verifyJWT } from "./helper/jwt";
import nanoid = require("nanoid");
@ -61,14 +61,22 @@ export class ConnectionManager {
}
const answer = (id: string, data: any, error: boolean = false) => {
if (error)
Logging.error(error as any);
socket.send(JSON.stringify({ ns: "message", data: { id, error, data } }));
}
const handler = new Map<string, ((data: any) => void)>();
handler.set("v2", async ({ id, query }: { id: string, query: IQuery }) => db.run(query, session)
handler.set("v2", async ({ id, query }) => db.run(Array.isArray(query) ? query : [query], session)
.then(res => answer(id, res))
.catch(err => answer(id, undefined, err)));
.catch(err => answer(id, undefined, err))
);
// handler.set("bulk", async ({ id, query }) => db.run(query, session)
// .then(res => answer(id, res))
// .catch(err => answer(id, undefined, err))
// );
const SnapshotMap = new Map<string, string>();
@ -106,10 +114,8 @@ export class ConnectionManager {
socket.on("close", () => {
Logging.log(`${session.id} has disconnected!`);
session.queries.forEach((query: DocumentQuery | CollectionQuery) => {
query.unsubscribe();
})
session.queries.clear();
session.subscriptions.forEach(unsubscribe => unsubscribe());
session.subscriptions.clear();
socket.removeAllListeners();
})
}

275
src/database/database.ts
View File

@ -1,29 +1,18 @@
import { Rules } from "./rules";
import Settings from "../settings";
import getLevelDB, { LevelDB, deleteLevelDB } from "../storage";
import getLevelDB, { LevelDB, deleteLevelDB, resNull } from "../storage";
import DocumentLock from "./lock";
import { DocumentQuery, CollectionQuery, Query, QueryError } from "./query";
import { DocumentQuery, CollectionQuery, Query, QueryError, ITypedQuery, IQuery } from "./query";
import Logging from "@hibas123/nodelogging";
import Session from "./session";
import nanoid = require("nanoid");
import nanoid = require("nanoid/generate");
import { Observable } from "@hibas123/utils";
type IWriteQueries = "set" | "update" | "delete" | "add";
type ICollectionQueries = "get" | "add" | "keys" | "delete-collection" | "list";
type IDocumentQueries = "get" | "set" | "update" | "delete";
export interface ITypedQuery<T> {
path: string[];
type: T;
data?: any;
options?: any;
}
interface ITransaction {
queries: ITypedQuery<IWriteQueries>[];
}
export type IQuery = ITypedQuery<ICollectionQueries | IDocumentQueries>;
const ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
// interface ITransaction {
// queries: ITypedQuery<IWriteQueries>[];
// }
export class DatabaseManager {
static databases = new Map<string, Database>();
@ -66,12 +55,17 @@ export type ChangeTypes = "added" | "modified" | "deleted";
export type Change = {
data: any;
document: string;
collection: string;
type: ChangeTypes;
sender: string;
}
export class Database {
public static getKey(collectionid: string, documentid?: string) {
return `${collectionid || ""}/${documentid || ""}`;
}
private level = getLevelDB(this.name);
get data() {
@ -84,10 +78,15 @@ export class Database {
public rules: Rules;
public locks = new DocumentLock()
private locks = new DocumentLock()
public collectionLocks = new DocumentLock()
public changes = new Map<string, Set<(change: Change) => void>>();
public changeListener = new Map<string, Set<(change: Change[]) => void>>();
public collectionChangeListener = new Observable<{
key: string;
id: string;
type: "create" | "delete"
}>();
toJSON() {
return {
@ -124,14 +123,71 @@ export class Database {
this.publickey = key;
}
public async resolve(path: string[], create = false): Promise<{ collection: string, document: string, collectionKey: string }> {
path = [...path]; // Create modifiable copy
let collectionID: string = undefined;
let documentKey = path.length % 2 === 0 ? path.pop() : undefined;
let key = path.join("/");
getQuery(path: string[], session: Session, type: "document" | "collection" | "any") {
if (type === "document")
return new DocumentQuery(this, path, session);
else if (type === "collection")
return new CollectionQuery(this, path, session);
else
return new Query(this, path, session);
const lock = await this.collectionLocks.lock(key);
try {
collectionID = await this.collections.get(key).then(r => r.toString()).catch(resNull);
if (!collectionID && create) {
collectionID = nanoid(ALPHABET, 32);
await this.collections.put(key, collectionID);
setImmediate(() => {
this.collectionChangeListener.send({
id: collectionID,
key,
type: "create"
})
})
}
} finally {
lock();
}
return {
collection: collectionID,
document: documentKey,
collectionKey: key
};
}
private sendChanges(changes: Change[]) {
let col = new Map<string, Map<string, Change[]>>();
changes.forEach(change => {
let e = col.get(change.collection);
if (!e) {
e = new Map()
col.set(change.collection, e);
}
let d = e.get(change.document);
if (!d) {
d = [];
e.set(change.document, d);
}
d.push(change);
})
setImmediate(() => {
for (let [collection, documents] of col.entries()) {
let collectionChanges = [];
for (let [document, documentChanges] of documents.entries()) {
let s = this.changeListener.get(Database.getKey(collection, document));
if (s)
s.forEach(e => setImmediate(() => e(documentChanges)));
collectionChanges.push(...documentChanges);
}
let s = this.changeListener.get(Database.getKey(collection))
if (s)
s.forEach(e => setImmediate(() => e(collectionChanges)))
}
})
}
private validate(query: ITypedQuery<any>) {
@ -146,80 +202,121 @@ export class Database {
throw inv;
}
async run(query: IQuery, session: Session) {
this.validate(query);
const isCollection = query.path.length % 2 === 1;
if (isCollection) {
const q = new CollectionQuery(this, query.path, session);
let type = query.type as ICollectionQueries;
switch (type) {
case "add":
return q.add(query.data);
case "get":
const limit = (query.options || {}).limit;
if (limit)
q.limit = limit;
const where = (query.options || {}).where;
if (where)
q.where = where;
return q.get();
case "keys":
return q.keys();
case "list":
return q.collections();
case "delete-collection":
return q.deleteCollection();
default:
return Promise.reject(new Error("Invalid query!"));
}
} else {
const q = new DocumentQuery(this, query.path, session);
let type = query.type as IDocumentQueries;
switch (type) {
case "get":
return q.get();
case "set":
return q.set(query.data, query.options || {});
case "update":
return q.update(query.data);
case "delete":
return q.delete();
default:
return Promise.reject(new Error("Invalid query!"));
async run(queries: IQuery[], session: Session) {
let resolve: { path: string[], create: boolean, resolved?: [string, string, string] }[] = [];
const addToResolve = (path: string[], create?: boolean) => {
let entry = resolve.find(e => { //TODO: Find may be slow...
if (e.path.length !== path.length)
return false;
for (let i = 0; i < e.path.length; i++) {
if (e.path[i] !== path[i])
return false;
}
return true;
})
if (!entry) {
entry = {
path,
create
}
resolve.push(entry);
}
entry.create = entry.create || create;
return entry;
}
const isBatch = queries.length > 1;
let parsed = queries.map(rawQuery => {
this.validate(rawQuery);
const isCollection = rawQuery.path.length % 2 === 1;
let query = isCollection
? new CollectionQuery(this, session, rawQuery)
: new DocumentQuery(this, session, rawQuery);
if (isBatch && !query.batchCompatible)
throw new Error("There are queries that are not batch compatible!");
let path = addToResolve(rawQuery.path, query.createCollection);
if (query.additionalLock)
addToResolve(query.additionalLock);
return {
path,
query
};
});
resolve = resolve.sort((a, b) => a.path.length - b.path.length);
let locks: (() => void)[] = [];
for (let e of resolve) {
let { collection, document, collectionKey } = await this.resolve(e.path, e.create);
e.resolved = [collection, document, collectionKey];
locks.push(
await this.locks.lock(collection, document)
);
}
let result = [];
try {
let batch = this.data.batch();
let changes: Change[] = [];
for (let e of parsed) {
result.push(
await e.query.run(e.path.resolved[0], e.path.resolved[1], batch, e.path.resolved[2])
);
changes.push(...e.query.changes);
}
if (batch.length > 0)
await batch.write();
this.sendChanges(changes);
} finally {
locks.forEach(lock => lock());
}
if (isBatch)
return result;
else
return result[0]
}
async snapshot(query: ITypedQuery<"snapshot">, session: Session, onchange: (change: any) => void) {
this.validate(query);
async snapshot(rawQuery: ITypedQuery<"snapshot">, session: Session, onchange: (change: any) => void) {
Logging.debug("Snaphot request:", rawQuery.path);
this.validate(rawQuery);
const isCollection = query.path.length % 2 === 1;
let q: DocumentQuery | CollectionQuery;
if (isCollection) {
q = new CollectionQuery(this, query.path, session);
const limit = (query.options || {}).limit;
if (limit)
q.limit = limit;
const where = (query.options || {}).where;
if (where)
q.where = where;
} else {
q = new DocumentQuery(this, query.path, session);
}
if (rawQuery.type !== "snapshot")
throw new Error("Invalid query type!");
const id = nanoid(16);
session.queries.set(id, q);
const isCollection = rawQuery.path.length % 2 === 1;
let query = isCollection
? new CollectionQuery(this, session, rawQuery, true)
: new DocumentQuery(this, session, rawQuery, true);
const {
unsubscribe,
value
} = await query.snapshot(onchange);
const id = nanoid(ALPHABET, 16);
session.subscriptions.set(id, unsubscribe);
return {
id,
snaphot: await q.snapshot(onchange)
snaphot: value
};
}
async unsubscribe(id: string, session: Session) {
let query: CollectionQuery | DocumentQuery = session.queries.get(id) as any;
let query = session.subscriptions.get(id);
if (query) {
query.unsubscribe();
session.queries.delete(id);
query();
session.subscriptions.delete(id);
}
}

1
src/database/lock.ts
View File

@ -8,6 +8,7 @@ export default class DocumentLock {
}
async lock(collection: string = "", document: string = "") {
//TODO: Check collection locks
let key = collection + "/" + document;
let l = this.locks.get(key);
if (l)

602
src/database/query.ts
View File

@ -4,6 +4,20 @@ import nanoid = require("nanoid/generate");
import Logging from "@hibas123/nodelogging";
import * as MSGPack from "what-the-pack";
import Session from "./session";
import { LevelUpChain } from "levelup";
export type IWriteQueries = "set" | "update" | "delete" | "add";
export type ICollectionQueries = "get" | "add" | "keys" | "delete-collection" | "list";
export type IDocumentQueries = "get" | "set" | "update" | "delete";
export interface ITypedQuery<T> {
path: string[];
type: T;
data?: any;
options?: any;
}
export type IQuery = ITypedQuery<ICollectionQueries | IDocumentQueries | "snapshot">;
export const MP = MSGPack.initialize(2 ** 20);
@ -11,7 +25,22 @@ const ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
const { encode, decode } = MP;
export class Query {
type Runner = (collection: string, document: string, batch: LevelUpChain, collectionKey: string) => any;
interface IPreparedQuery {
createCollection: boolean;
needDocument: boolean;
batchCompatible: boolean;
runner: Runner;
additionalLock?: string[];
}
interface DocRes {
id: string;
data: any;
}
export abstract class Query {
/**
* Returns true if the path only contains valid characters and false if it doesn't
* @param path Path to be checked
@ -20,48 +49,37 @@ export class Query {
return path.every(e => (e.match(/[^a-zA-Z0-9_\-\<\>]/g) || []).length === 0);
}
constructor(protected database: Database, protected path: string[], protected session: Session) {
if (path.length > 10) {
public changes: Change[] = [];
public readonly createCollection: boolean;
public readonly needDocument: boolean;
public readonly batchCompatible: boolean;
public readonly additionalLock?: string[];
private readonly _runner: Runner;
constructor(protected database: Database, protected session: Session, protected query: IQuery, snapshot = false) {
if (query.path.length > 10) {
throw new QueryError("Path is to long. Path is only allowed to be 10 Layers deep!");
}
if (!this.validatePath(path)) {
if (!this.validatePath(query.path)) {
throw new QueryError("Path can only contain a-z A-Z 0-9 '-' '-' '<' and '>' ");
}
}
protected async resolve(path: string[], create = false): Promise<{ collection: string, document: string, collectionKey: string }> {
path = [...path]; // Create modifiable copy
let collectionID: string = undefined;
let documentKey = path.length % 2 === 0 ? path.pop() : undefined;
let key = path.join("/");
const lock = await this.database.collectionLocks.lock(key);
try {
collectionID = await this.database.collections.get(key).then(r => r.toString()).catch(resNull);
if (!collectionID && create) {
collectionID = nanoid(ALPHABET, 32);
await this.database.collections.put(key, collectionID);
}
} finally {
lock();
if (!snapshot) {
let data = this.prepare(query);
this.createCollection = data.createCollection;
this.needDocument = data.needDocument;
this.batchCompatible = data.batchCompatible;
this.additionalLock = data.additionalLock;
this._runner = data.runner;
}
return {
collection: collectionID,
document: documentKey,
collectionKey: key
};
}
protected getKey(collection: string, document?: string) {
return `${collection || ""}/${document || ""}`;
}
protected abstract prepare(query: IQuery): IPreparedQuery;
protected getDoc(collection: string, document: string) {
return this.database.data
.get(this.getKey(collection, document), { asBuffer: true })
.get(Database.getKey(collection, document), { asBuffer: true })
.then(res => decode<any>(res as Buffer)).catch(resNull);
}
@ -69,22 +87,90 @@ export class Query {
let change: Change = {
type,
document,
collection,
data,
sender: this.session.id
}
let s = this.database.changes.get(this.getKey(collection, document))
if (s)
s.forEach(e => setImmediate(() => e(change)))
s = this.database.changes.get(this.getKey(collection))
if (s)
s.forEach(e => setImmediate(() => e(change)))
this.changes.push(change);
}
protected static getConstructorParams(query: Query): [Database, string[], Session] {
return [query.database, query.path, query.session];
protected static getConstructorParams(query: Query): [Database, Session, IQuery] {
return [query.database, query.session, query.query];
}
protected abstract checkChange(change: Change): boolean;
protected abstract firstSend(collection: string, document: string): Promise<any>;
public run(collection: string, document: string, batch: LevelUpChain, collectionKey: string) {
return this._runner.call(this, collection, document, batch, collectionKey);
}
public async snapshot(onChange: (change: (DocRes & { type: ChangeTypes })[]) => void) {
const receivedChanges = (changes: Change[]) => {
let res = changes.filter(change => this.checkChange(change)).map(change => {
return {
id: change.document,
data: change.data,
type: change.type
}
})
if (res.length > 0)
onChange(res);
};
const unsub = this.database.collectionChangeListener.subscribe(change => {
if (change.key === collectionKey) {
if (change.type === "create")
addSubscriber(change.id);
else
removeSubscriber(); // Send delete for all elements (Don't know how to do this...)
}
})
let { collection, document, collectionKey } = await this.database.resolve(this.query.path)
let oldKey: string = undefined;
const removeSubscriber = () => {
if (!oldKey)
return;
let s = this.database.changeListener.get(oldKey);
if (s) {
s.delete(receivedChanges);
if (s.size <= 0)
this.database.changeListener.delete(oldKey);
}
oldKey = undefined;
}
const addSubscriber = (collection: string) => {
let key = Database.getKey(collection, document);
if (oldKey !== key) {
if (oldKey !== undefined)
removeSubscriber();
let s = this.database.changeListener.get(key);
if (!s) {
s = new Set();
this.database.changeListener.set(key, s);
}
s.add(receivedChanges);
}
}
if (collection) {
addSubscriber(collection);
}
return {
unsubscribe: () => {
unsub();
removeSubscriber();
},
value: await this.firstSend(collection, document)
}
}
}
@ -95,185 +181,140 @@ interface UpdateData {
}
}
export class DocumentQuery extends Query {
constructor(database: Database, path: string[], session: Session) {
super(database, path, session);
this.onChange = this.onChange.bind(this);
prepare(query: IQuery): IPreparedQuery {
let type = query.type as IDocumentQueries;
switch (type) {
case "get":
return {
batchCompatible: false,
createCollection: false,
needDocument: false,
runner: this.get
}
case "set":
return {
batchCompatible: true,
createCollection: true,
needDocument: true,
runner: this.set
}
case "update":
return {
batchCompatible: true,
createCollection: true,
needDocument: true,
runner: this.update
}
case "delete":
return {
batchCompatible: true,
createCollection: false,
needDocument: true,
runner: this.delete
}
default:
throw new Error("Invalid query type: " + type);
}
}
public async get() {
let { collection, document } = await this.resolve(this.path);
private async get(collection: string, document: string) {
if (!collection || !document) {
return null;
}
return this.getDoc(collection, document)
return this.getDoc(collection, document);
}
public async set(data: any, { merge = false }) {
private async set(collection: string, document: string, batch?: LevelUpChain) {
const { data, options } = this.query;
if (data === null)
return this.delete();
let { collection, document } = await this.resolve(this.path, true);
if (!collection) {
throw new QueryError("There must be a collection!")
}
return this.delete(collection, document, batch);
if (!document) {
throw new QueryError("There must be a document key!")
}
const lock = await this.database.locks.lock(collection, document);
let isNew = !(await this.getDoc(collection, document))
return this.database.data
.put(this.getKey(collection, document), encode(data))
.then(() => this.sendChange(collection, document, isNew ? "added" : "modified", data))
.finally(() => lock())
batch.put(Database.getKey(collection, document), encode(data))
this.sendChange(collection, document, isNew ? "added" : "modified", data)
}
public async update(updateData: UpdateData) {
let { collection, document } = await this.resolve(this.path, true);
if (!collection) {
throw new QueryError("There must be a collection!")
}
if (!document) {
throw new QueryError("There must be a document key!")
}
// Logging.debug(updateData);
const lock = await this.database.locks.lock(collection, document);
try {
let data = await this.getDoc(collection, document);
let isNew = false
if (!data) {
isNew = true;
data = {};
}
for (let path in updateData) {
const toUpdate = updateData[path];
let d = data;
let parts = path.split(".");
while (parts.length > 1) {
let seg = parts.shift();
if (!data[seg])
data[seg] = {}
d = data[seg];
}
const last = parts[0];
// Logging.debug(parts, last, d)
switch (toUpdate.type) {
case "value":
d[last] = toUpdate.value;
break;
case "increment":
if (d[last] === undefined || d[last] === null)
d[last] = toUpdate.value;
else if (typeof d[last] !== "number") {
throw new QueryError("Field is no number!");
} else {
d[last] += toUpdate.value;
}
break;
case "timestamp":
d[last] = new Date().valueOf();
break;
case "push":
if (d[last] === undefined || d[last] === null)
d[last] = [toUpdate.value];
else if (Array.isArray(d[last])) {
d[last].push(toUpdate.value);
} else {
throw new QueryError("Field is not array!");
}
break;
default:
throw new QueryError("Invalid update type: " + toUpdate.type);
}
}
this.database.data
.put(this.getKey(collection, document), encode(data))
.then(() => this.sendChange(collection, document, isNew ? "added" : "modified", data))
} finally {
lock();
}
//TODO: Implement
}
public async delete() {
let { collection, document } = await this.resolve(this.path);
if (!collection) {
throw new QueryError("There must be a collection!")
}
if (!document) {
throw new QueryError("There must be a document key!")
}
const lock = await this.database.locks.lock(collection, document);
return await this.database.data
.del(`${collection}/${document}`)
.then(() => this.sendChange(collection, document, "deleted", null))
.finally(() => lock())
}
private subscription: {
key: string,
onChange: (change: DocRes & { type: ChangeTypes }) => void
};
async snapshot(onChange: (change: DocRes & { type: ChangeTypes }) => void) {
if (this.subscription)
throw new QueryError("This query is already subscribed!");
let { collection, document } = await this.resolve(this.path);
private async update(collection: string, document: string, batch?: LevelUpChain) {
const updateData: UpdateData = this.query.data;
let data = await this.getDoc(collection, document);
let key = this.getKey(collection, document);
this.subscription = {
key,
onChange
}
let s = this.database.changes.get(key);
if (!s) {
s = new Set();
this.database.changes.set(key, s);
let isNew = false
if (!data) {
isNew = true;
data = {};
}
s.add(this.onChange);
for (let path in updateData) {
const toUpdate = updateData[path];
let d = data;
let parts = path.split(".");
while (parts.length > 1) {
let seg = parts.shift();
if (!data[seg])
data[seg] = {}
d = data[seg];
}
return data;
const last = parts[0];
switch (toUpdate.type) {
case "value":
d[last] = toUpdate.value;
break;
case "increment":
if (d[last] === undefined || d[last] === null)
d[last] = toUpdate.value;
else if (typeof d[last] !== "number") {
throw new QueryError("Field is no number!");
} else {
d[last] += toUpdate.value;
}
break;
case "timestamp":
d[last] = new Date().valueOf();
break;
case "push":
if (d[last] === undefined || d[last] === null)
d[last] = [toUpdate.value];
else if (Array.isArray(d[last])) {
d[last].push(toUpdate.value);
} else {
throw new QueryError("Field is not array!");
}
break;
default:
throw new QueryError("Invalid update type: " + toUpdate.type);
}
}
if (batch) {
batch.put(Database.getKey(collection, document), encode(data))
} else {
await this.database.data
.put(Database.getKey(collection, document), encode(data))
}
this.sendChange(collection, document, isNew ? "added" : "modified", data)
}
onChange(change: Change) {
// if(change.sender === this.sender)
// return
this.subscription.onChange({
id: change.document,
data: change.data,
type: change.type
})
private async delete(collection: string, document: string, batch?: LevelUpChain) {
if (batch) {
batch.del(Database.getKey(collection, document))
} else {
await this.database.data.del(Database.getKey(collection, document));
}
this.sendChange(collection, document, "deleted", null)
}
unsubscribe() {
if (!this.subscription)
return;
let s = this.database.changes.get(this.subscription.key);
s.delete(this.onChange);
if (s.size <= 0)
this.database.changes.delete(this.subscription.key);
checkChange(change: Change) {
return true;
}
this.subscription = undefined;
firstSend(collection: string, document: string) {
return this.get(collection, document);
}
public static fromQuery(query: Query) {
@ -302,15 +343,61 @@ type IQueryWhereArray = [FieldPath, WhereFilterOp, any];
type IQueryWhere = IQueryWhereArray | IQueryWhereVerbose;
interface DocRes {
id: string;
data: any;
}
export class CollectionQuery extends Query {
constructor(database: Database, path: string[], session: Session) {
super(database, path, session);
this.onChange = this.onChange.bind(this);
private _addId: string;
prepare(query): IPreparedQuery {
switch (query.type as ICollectionQueries) {
case "add":
this._addId = nanoid(ALPHABET, 32)
return {
batchCompatible: true,
createCollection: true,
needDocument: false,
runner: this.add,
additionalLock: [...query.path, this._addId]
}
case "get":
const limit = (query.options || {}).limit;
if (limit)
this.limit = limit;
const where = (query.options || {}).where;
if (where)
this.where = where;
return {
batchCompatible: false,
createCollection: false,
needDocument: false,
runner: this.get
}
case "keys":
return {
batchCompatible: false,
createCollection: false,
needDocument: false,
runner: this.keys
}
case "list":
return {
batchCompatible: false,
createCollection: false,
needDocument: false,
runner: this.keys
}
case "delete-collection":
return {
batchCompatible: false,
createCollection: false,
needDocument: false,
runner: this.deleteCollection
}
// run = () => q.deleteCollection();
// break;
default:
throw new Error("Invalid query!");
}
}
@ -338,15 +425,19 @@ export class CollectionQuery extends Query {
public limit: number = -1;
public async add(value: any) {
let id = nanoid(ALPHABET, 32);
let q = new DocumentQuery(this.database, [...this.path, id], this.session);
await q.set(value, {});
return id;
public async add(collection: string, document: string, batch: LevelUpChain, collectionKey: string) {
let q = new DocumentQuery(this.database, this.session, {
type: "set",
path: this.additionalLock,
data: this.query.data,
options: this.query.options
});
await q.run(collection, this._addId, batch, collectionKey);
return this._addId;
}
private getStreamOptions(collection: string) {
let gt = Buffer.from(this.getKey(collection) + " ");
let gt = Buffer.from(Database.getKey(collection) + " ");
gt[gt.length - 1] = 0;
let lt = Buffer.alloc(gt.length);
@ -359,10 +450,7 @@ export class CollectionQuery extends Query {
}
}
public async keys() {
let { collection, document } = await this.resolve(this.path);
if (document)
throw new QueryError("Keys only works on collections!");
public async keys(collection: string) {
if (!collection)
return []
@ -382,7 +470,7 @@ export class CollectionQuery extends Query {
});
}
private getFieldValue(data: any, path: FieldPath) {
private _getFieldValue(data: any, path: FieldPath) {
let parts = path.split(".");
let d = data;
while (parts.length > 0) {
@ -395,10 +483,10 @@ export class CollectionQuery extends Query {
return d;
}
private fitsWhere(data: any): boolean {
private _fitsWhere(data: any): boolean {
if (this._where.length > 0) {
return this._where.every(([fieldPath, opStr, value]) => {
let val = this.getFieldValue(data, fieldPath);
let val = this._getFieldValue(data, fieldPath);
switch (opStr) {
case "<":
return val < value;
@ -430,10 +518,7 @@ export class CollectionQuery extends Query {
return true;
}
async get() {
let { collection, document } = await this.resolve(this.path);
if (document)
throw new QueryError("Keys only works on collections!");
async get(collection: string) {
if (!collection)
return [];
@ -464,7 +549,7 @@ export class CollectionQuery extends Query {
const id = s[1];
let data = decode(value);
if (this.fitsWhere(data)) {
if (this._fitsWhere(data)) {
if (this.limit < 0 || values.length < this.limit) {
values.push({
id,
@ -486,59 +571,14 @@ export class CollectionQuery extends Query {
})
}
private subscription: {
key: string,
onChange: (change: (DocRes & { type: ChangeTypes })[]) => void
};
async snapshot(onChange: (change: (DocRes & { type: ChangeTypes })[]) => void) {
if (this.subscription)
throw new QueryError("This query is already subscribed!");
let { collection, document } = await this.resolve(this.path, true);
let data = await this.get();
let key = this.getKey(collection, document);
this.subscription = {
key,
onChange
}
let s = this.database.changes.get(key);
if (!s) {
s = new Set();
this.database.changes.set(key, s);
}
s.add(this.onChange);
return data;
checkChange(change: Change) {
return this._fitsWhere(change.data);
}
onChange(change: Change) {
// if(change.sender === this.sender)
// return
if (this.fitsWhere(change.data)) {
this.subscription.onChange([{
id: change.document,
data: change.data,
type: change.type
}])
}
firstSend(collection: string) {
return this.get(collection)
}
unsubscribe() {
if (!this.subscription)
return;
let s = this.database.changes.get(this.subscription.key);
s.delete(this.onChange);
if (s.size <= 0)
this.database.changes.delete(this.subscription.key);
this.subscription = undefined;
}
public async collections() {
if (!this.session.root)
throw new QueryError("No Permission!");
@ -552,29 +592,27 @@ export class CollectionQuery extends Query {
});
}
public async deleteCollection() {
public async deleteCollection(collection: string, document: string, _b: LevelUpChain, collectionKey: string) {
if (!this.session.root)
throw new QueryError("No Permission!");
const { collection, document, collectionKey } = await this.resolve(this.path);
if (document) {
throw new QueryError("There can be no document defined on this operation");
}
//TODO: Lock whole collection!
let batch = this.database.data.batch();
try {
if (collection) {
let documents = await this.keys();
let documents = await this.keys(collection);
// Logging.debug("To delete:", documents)
for (let document of documents) {
batch.del(this.getKey(collection, document));
batch.del(Database.getKey(collection, document));
}
await batch.write();
batch = undefined;
await this.database.collections.del(collectionKey);
this.database.collectionChangeListener.send({
id: collection,
key: collectionKey,
type: "delete"
});
}
} finally {
if (batch)

3
src/database/session.ts
View File

@ -1,4 +1,3 @@
import { Query } from "./query";
export default class Session {
constructor(private _sessionid: string) { }
@ -8,5 +7,5 @@ export default class Session {
root: boolean = false;
uid: string = undefined;
queries = new Map<string, Query>();
subscriptions = new Map<string, (() => void)>();
}

2
src/web/v1/index.ts
View File

@ -49,7 +49,7 @@ V1.post("/db/:database/query", async ctx => {
}
}
ctx.body = await db.run(query, session).catch(err => {
ctx.body = await db.run([query], session).catch(err => {
if (err instanceof QueryError) {
throw new BadRequestError(err.message);
}