Fixing bug on collection deletion

Extending Admin Interface
Adding cleanup procedure, that clears undeleted collection data

src/database/database.ts

@@ -3,6 +3,7 @@ import Settings from "../settings";
 import getLevelDB, { LevelDB, deleteLevelDB } from "../storage";
 import DocumentLock from "./lock";
 import { DocumentQuery, CollectionQuery, Query } from "./query";
+import Logging from "@hibas123/nodelogging";
 
 export class DatabaseManager {
    static databases = new Map<string, Database>();
@@ -116,4 +117,105 @@ export class Database {
    async stop() {
       await this.data.close();
    }
+
+   public async runCleanup() {
+      const should = await new Promise<Set<string>>((yes, no) => {
+         const stream = this.collections.iterator({
+            keyAsBuffer: false,
+            valueAsBuffer: false
+         })
+
+         const collections = new Set<string>();
+         const onValue = (err: Error, key: string, value: string) => {
+            if (err) {
+               Logging.error(err);
+               stream.end((err) => Logging.error(err))
+               no(err);
+            }
+
+            if (!key && !value) {
+               yes(collections);
+            } else {
+               collections.add(value)
+               stream.next(onValue);
+            }
+         }
+
+         stream.next(onValue);
+      })
+
+      const existing = await new Promise<Set<string>>((yes, no) => {
+         const stream = this.data.iterator({
+            keyAsBuffer: false,
+            values: false
+         })
+
+         const collections = new Set<string>();
+         const onValue = (err: Error, key: string, value: Buffer) => {
+            if (err) {
+               Logging.error(err);
+               stream.end((err) => Logging.error(err))
+               no(err);
+            }
+
+            if (!key && !value) {
+               yes(collections);
+            } else {
+               let coll = key.split("/")[0];
+               collections.add(coll)
+               stream.next(onValue);
+            }
+         }
+
+         stream.next(onValue);
+      })
+
+      const toDelete = new Set<string>();
+      existing.forEach(collection => {
+         if (!should.has(collection))
+            toDelete.add(collection);
+      })
+
+      for (let collection of toDelete) {
+         const batch = this.data.batch();
+
+         let gt = Buffer.from(collection + "/ ");
+         gt[gt.length - 1] = 0;
+
+         let lt = Buffer.alloc(gt.length);
+         lt.set(gt);
+         lt[gt.length - 1] = 0xFF;
+
+         await new Promise<void>((yes, no) => {
+            const stream = this.data.iterator({
+               keyAsBuffer: false,
+               values: false,
+               gt,
+               lt
+            })
+
+            const onValue = (err: Error, key: string, value: Buffer) => {
+               if (err) {
+                  Logging.error(err);
+                  stream.end((err) => Logging.error(err))
+                  no(err);
+               }
+
+               if (!key && !value) {
+                  yes();
+               } else {
+                  batch.del(key);
+                  stream.next(onValue);
+               }
+            }
+
+            stream.next(onValue);
+         })
+
+         await batch.write();
+      }
+
+      return Array.from(toDelete.values());
+   }
+
 }

src/database/query.ts

@@ -539,7 +539,8 @@ export class CollectionQuery extends Query {
       try {
          if (collection) {
             let documents = await this.keys();
-            for (let document in documents) {
+            // Logging.debug("To delete:", documents)
+            for (let document of documents) {
                batch.del(this.getKey(collection, document));
             }
             await batch.write();

src/web/helper/form.ts

@@ -1,4 +1,4 @@
-import getTemplate from "./hb";
+import { getTemplate } from "./hb";
 import { Context } from "vm";
 
 interface IFormConfigField {

src/web/helper/hb.ts

@@ -37,8 +37,22 @@ Handlebars.registerHelper('ifCond', function (v1, operator, v2, options) {
 
 
 const cache = new Map<string, Handlebars.TemplateDelegate>();
+const htmlCache = new Map<string, string>();
 
-export default function getTemplate(name: string) {
+export function getView(name: string) {
+   let tl: string;
+   if (!config.dev)
+      tl = htmlCache.get(name);
+
+   if (!tl) {
+      tl = readFileSync(`./views/${name}.html`).toString();
+      htmlCache.set(name, tl);
+   }
+
+   return tl;
+}
+
+export function getTemplate(name: string) {
    let tl: Handlebars.TemplateDelegate;
    if (!config.dev)
       tl = cache.get(name);

src/web/helper/table.ts

@@ -1,5 +1,5 @@
 import { Context } from "koa";
-import getTemplate from "./hb";
+import { getTemplate } from "./hb";
 
 export default function getTable(title: string, data: any[], ctx: Context) {
    let table: string[][] = [];

src/web/v1/admin.ts

@@ -7,6 +7,7 @@ import { DatabaseManager } from "../../database/database";
 import { MP } from "../../database/query";
 import config from "../../config";
 import Logging from "@hibas123/logging";
+import { getView } from "../helper/hb";
 
 const AdminRoute = new Router();
 
@@ -17,6 +18,11 @@ AdminRoute.use(async (ctx, next) => {
    return next();
 })
 
+AdminRoute.get("/", async ctx => {
+   //TODO: Main Interface
+   ctx.body = getView("admin");
+});
+
 AdminRoute.get("/settings", async ctx => {
    let res = await new Promise<string[][]>((yes, no) => {
       const stream = Settings.db.createReadStream({
@@ -51,11 +57,11 @@ AdminRoute.get("/data", async ctx => {
          keys: true,
          values: true,
          valueAsBuffer: true,
-         keyAsBuffer: false
+         keyAsBuffer: false,
+         limit: 1000
       });
       let res = [["key", "value"]];
       stream.on("data", ({ key, value }: { key: string, value: Buffer }) => {
-         Logging.debug("Admin Key:", key);
          res.push([key, key.split("/").length > 2 ? value.toString() : JSON.stringify(MP.decode(value))]);
       })
 
@@ -114,6 +120,48 @@ AdminRoute
       ctx.body = "Success";
    })
 
+AdminRoute.get("/collections", async ctx => {
+   const { database } = ctx.query;
+   let db = DatabaseManager.getDatabase(database);
+   if (!db)
+      throw new BadRequestError("Database not found");
+
+   let res = await new Promise<string[]>((yes, no) => {
+
+      const stream = db.collections.createKeyStream({
+         keyAsBuffer: false,
+         limit: 1000
+      });
+      let res = [];
+      stream.on("data", (key: string) => {
+         res.push(key);
+      })
+
+      stream.on("error", no);
+      stream.on("end", () => yes(res))
+   })
+
+   if (ctx.query.view) {
+      return getTable("Databases", res, ctx);
+   } else {
+      ctx.body = res;
+   }
+})
+
+AdminRoute.get("/collections/cleanup", async ctx => {
+   const { database } = ctx.query;
+   let db = DatabaseManager.getDatabase(database);
+   if (!db)
+      throw new BadRequestError("Database not found");
+
+   let deleted = await db.runCleanup();
+   if (ctx.query.view) {
+      return getTable("Databases", deleted, ctx);
+   } else {
+      ctx.body = deleted;
+   }
+})
+
 AdminRoute.get("/database/new", getForm("/v1/admin/database", "New/Change Database", {
    name: { label: "Name", type: "text", },
    accesskey: { label: "Access Key", type: "text" },