Fixing bug on collection deletion

Extending Admin Interface
Adding cleanup procedure, that clears undeleted collection data

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hibas123/realtimedb",
-  "version": "2.0.0-beta.3",
+  "version": "2.0.0-beta.4",
   "description": "",
   "main": "lib/index.js",
   "private": true,

src/database/database.ts

@@ -3,6 +3,7 @@ import Settings from "../settings";
 import getLevelDB, { LevelDB, deleteLevelDB } from "../storage";
 import DocumentLock from "./lock";
 import { DocumentQuery, CollectionQuery, Query } from "./query";
+import Logging from "@hibas123/nodelogging";
 
 export class DatabaseManager {
    static databases = new Map<string, Database>();
@@ -116,4 +117,105 @@ export class Database {
    async stop() {
       await this.data.close();
    }
+
+   public async runCleanup() {
+      const should = await new Promise<Set<string>>((yes, no) => {
+         const stream = this.collections.iterator({
+            keyAsBuffer: false,
+            valueAsBuffer: false
+         })
+
+         const collections = new Set<string>();
+         const onValue = (err: Error, key: string, value: string) => {
+            if (err) {
+               Logging.error(err);
+               stream.end((err) => Logging.error(err))
+               no(err);
+            }
+
+            if (!key && !value) {
+               yes(collections);
+            } else {
+               collections.add(value)
+               stream.next(onValue);
+            }
+         }
+
+         stream.next(onValue);
+      })
+
+      const existing = await new Promise<Set<string>>((yes, no) => {
+         const stream = this.data.iterator({
+            keyAsBuffer: false,
+            values: false
+         })
+
+         const collections = new Set<string>();
+         const onValue = (err: Error, key: string, value: Buffer) => {
+            if (err) {
+               Logging.error(err);
+               stream.end((err) => Logging.error(err))
+               no(err);
+            }
+
+            if (!key && !value) {
+               yes(collections);
+            } else {
+               let coll = key.split("/")[0];
+               collections.add(coll)
+               stream.next(onValue);
+            }
+         }
+
+         stream.next(onValue);
+      })
+
+      const toDelete = new Set<string>();
+      existing.forEach(collection => {
+         if (!should.has(collection))
+            toDelete.add(collection);
+      })
+
+      for (let collection of toDelete) {
+         const batch = this.data.batch();
+
+         let gt = Buffer.from(collection + "/ ");
+         gt[gt.length - 1] = 0;
+
+         let lt = Buffer.alloc(gt.length);
+         lt.set(gt);
+         lt[gt.length - 1] = 0xFF;
+
+         await new Promise<void>((yes, no) => {
+            const stream = this.data.iterator({
+               keyAsBuffer: false,
+               values: false,
+               gt,
+               lt
+            })
+
+            const onValue = (err: Error, key: string, value: Buffer) => {
+               if (err) {
+                  Logging.error(err);
+                  stream.end((err) => Logging.error(err))
+                  no(err);
+               }
+
+               if (!key && !value) {
+                  yes();
+               } else {
+                  batch.del(key);
+                  stream.next(onValue);
+               }
+            }
+
+            stream.next(onValue);
+         })
+
+         await batch.write();
+      }
+
+      return Array.from(toDelete.values());
+   }
+
 }

src/database/query.ts

@@ -539,7 +539,8 @@ export class CollectionQuery extends Query {
       try {
          if (collection) {
             let documents = await this.keys();
-            for (let document in documents) {
+            // Logging.debug("To delete:", documents)
+            for (let document of documents) {
                batch.del(this.getKey(collection, document));
             }
             await batch.write();

src/web/helper/form.ts

@@ -1,4 +1,4 @@
-import getTemplate from "./hb";
+import { getTemplate } from "./hb";
 import { Context } from "vm";
 
 interface IFormConfigField {

src/web/helper/hb.ts

@@ -37,8 +37,22 @@ Handlebars.registerHelper('ifCond', function (v1, operator, v2, options) {
 
 
 const cache = new Map<string, Handlebars.TemplateDelegate>();
+const htmlCache = new Map<string, string>();
 
-export default function getTemplate(name: string) {
+export function getView(name: string) {
+   let tl: string;
+   if (!config.dev)
+      tl = htmlCache.get(name);
+
+   if (!tl) {
+      tl = readFileSync(`./views/${name}.html`).toString();
+      htmlCache.set(name, tl);
+   }
+
+   return tl;
+}
+
+export function getTemplate(name: string) {
    let tl: Handlebars.TemplateDelegate;
    if (!config.dev)
       tl = cache.get(name);

src/web/helper/table.ts

@@ -1,5 +1,5 @@
 import { Context } from "koa";
-import getTemplate from "./hb";
+import { getTemplate } from "./hb";
 
 export default function getTable(title: string, data: any[], ctx: Context) {
    let table: string[][] = [];

src/web/v1/admin.ts

@@ -7,6 +7,7 @@ import { DatabaseManager } from "../../database/database";
 import { MP } from "../../database/query";
 import config from "../../config";
 import Logging from "@hibas123/logging";
+import { getView } from "../helper/hb";
 
 const AdminRoute = new Router();
 
@@ -17,6 +18,11 @@ AdminRoute.use(async (ctx, next) => {
    return next();
 })
 
+AdminRoute.get("/", async ctx => {
+   //TODO: Main Interface
+   ctx.body = getView("admin");
+});
+
 AdminRoute.get("/settings", async ctx => {
    let res = await new Promise<string[][]>((yes, no) => {
       const stream = Settings.db.createReadStream({
@@ -51,11 +57,11 @@ AdminRoute.get("/data", async ctx => {
          keys: true,
          values: true,
          valueAsBuffer: true,
-         keyAsBuffer: false
+         keyAsBuffer: false,
+         limit: 1000
       });
       let res = [["key", "value"]];
       stream.on("data", ({ key, value }: { key: string, value: Buffer }) => {
-         Logging.debug("Admin Key:", key);
          res.push([key, key.split("/").length > 2 ? value.toString() : JSON.stringify(MP.decode(value))]);
       })
 
@@ -114,6 +120,48 @@ AdminRoute
       ctx.body = "Success";
    })
 
+AdminRoute.get("/collections", async ctx => {
+   const { database } = ctx.query;
+   let db = DatabaseManager.getDatabase(database);
+   if (!db)
+      throw new BadRequestError("Database not found");
+
+   let res = await new Promise<string[]>((yes, no) => {
+
+      const stream = db.collections.createKeyStream({
+         keyAsBuffer: false,
+         limit: 1000
+      });
+      let res = [];
+      stream.on("data", (key: string) => {
+         res.push(key);
+      })
+
+      stream.on("error", no);
+      stream.on("end", () => yes(res))
+   })
+
+   if (ctx.query.view) {
+      return getTable("Databases", res, ctx);
+   } else {
+      ctx.body = res;
+   }
+})
+
+AdminRoute.get("/collections/cleanup", async ctx => {
+   const { database } = ctx.query;
+   let db = DatabaseManager.getDatabase(database);
+   if (!db)
+      throw new BadRequestError("Database not found");
+
+   let deleted = await db.runCleanup();
+   if (ctx.query.view) {
+      return getTable("Databases", deleted, ctx);
+   } else {
+      ctx.body = deleted;
+   }
+})
+
 AdminRoute.get("/database/new", getForm("/v1/admin/database", "New/Change Database", {
    name: { label: "Name", type: "text", },
    accesskey: { label: "Access Key", type: "text" },

views/admin.html

@@ -0,0 +1,112 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+   <meta charset="UTF-8">
+   <meta name="viewport" content="width=device-width, initial-scale=1.0">
+   <meta http-equiv="X-UA-Compatible" content="ie=edge">
+   <title>Admin Interface</title>
+   <link rel="stylesheet" href="https://unpkg.com/@hibas123/theme/out/base.css">
+   <link rel="stylesheet" href="https://unpkg.com/@hibas123/theme/out/light.css">
+
+   <script src="https://unpkg.com/handlebars/dist/handlebars.min.js"></script>
+
+   <style>
+      #message {
+         visibility: hidden;
+         background-color: lightgreen;
+         border: 1px solid lime;
+         border-radius: .5rem;
+         padding: 1rem;
+         font-size: 1.5rem;
+         margin-bottom: 1rem;
+      }
+
+      .grid {
+         display: grid;
+         height: 100vh;
+         grid-template-columns: 360px auto;
+      }
+
+      #content {
+         position: absolute;
+         top: 0;
+         left: 0;
+         width: 100%;
+         height: 100%;
+         border: 0;
+      }
+   </style>
+</head>
+
+<body>
+   <div class="grid">
+      <div style="border-right: 1px solid darkgrey; padding: 1rem;">
+         <h2>Navigation: </h2>
+         <ul class="list list-clickable">
+            <li onclick="loadView('settings');">Settings</li>
+            <li onclick="loadView('database', {full:true});">Databases</li>
+            <li onclick="loadView('database/new');">New Database</li>
+         </ul>
+         Databases:
+         <div id="dbs" class="list list-clickable" style="margin: 1rem;"></div>
+
+      </div>
+      <div style="position:relative;">
+         <iframe id="content"></iframe>
+      </div>
+   </div>
+
+   <template>
+
+   </template>
+
+   <script>
+      const key = new URL(window.location.href).searchParams.get("key");
+      const content = document.getElementById("content");
+      const base = new URL(window.location.href).host;
+
+      function getUrl(name, params, view = true) {
+         const url = new URL(window.location.href);
+         url.pathname = "/v1/admin/" + name;
+         for (let key in params || {})
+            url.searchParams.set(key, params[key]);
+
+         url.searchParams.set("key", key);
+         if (view)
+            url.searchParams.set("view", "true");
+
+         return url.href;
+      }
+
+      function loadView(name, params) {
+         content.src = getUrl(name, params);
+      }
+
+      loadView("settings")
+
+      const dbsul = document.getElementById("dbs");
+      function reloadDBs() {
+         fetch(getUrl("database", {}, false))
+            .then(res => res.json())
+            .then(databases => databases.map(database => `
+<div class="card margin elv-4">
+   <h3>${database}</h3>
+   <button class=btn onclick="loadView('data', {database:'${database}'})">Data</button>
+   <button class=btn onclick="loadView('collections', {database:'${database}'})">Collections</button>
+   <button class=btn onclick="loadView('collections/cleanup', {database:'${database}'})">Clean</button>
+</div>`
+            ))
+            .then(d => d.join("\n"))
+            .then(d => dbsul.innerHTML = d)
+            .catch(console.error)
+
+
+      }
+
+      reloadDBs();
+      setInterval(reloadDBs, 5000);
+   </script>
+</body>
+
+</html>