From 68011e3866ccbfd6741330f7eff9aac055385b31 Mon Sep 17 00:00:00 2001 From: Fabian Stamm Date: Mon, 4 Nov 2019 00:45:30 +0100 Subject: [PATCH] First basic version --- package-lock.json | 218 +++++++++++--------- package.json | 31 ++- src/connection.ts | 24 ++- src/database/database.ts | 39 ++-- src/database/lock.ts | 48 ++--- src/database/query.ts | 420 +++++++++++++++++++-------------------- src/database/session.ts | 4 + src/storage.ts | 17 +- src/types.d.ts | 10 + src/web/v1/admin.ts | 13 +- 10 files changed, 431 insertions(+), 393 deletions(-) create mode 100644 src/types.d.ts diff --git a/package-lock.json b/package-lock.json index b995491..29eb7b2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,29 +4,21 @@ "lockfileVersion": 1, "requires": true, "dependencies": { - "@hibas123/binary-encoder": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@hibas123/binary-encoder/-/binary-encoder-1.0.0.tgz", - "integrity": "sha512-H1SeL1bmLcnxRkOiOC4zyhy1toD/EMn7fP+UA7ydlPAHN097P8llF53asIzhlmgBUaCPNythqnLskXZtcC+v3A==", - "requires": { - "@hibas123/logging": "^2.1.0" - } - }, "@hibas123/logging": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@hibas123/logging/-/logging-2.1.0.tgz", - "integrity": "sha512-h8uZcFTxkbvEy3BgEYXl6ElfV5I1bonkYT6slBtzH38klkVv3GgkA7EL21JewPiYuNOsQYfXyXxoYxnxqWJgJA==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@hibas123/logging/-/logging-2.1.1.tgz", + "integrity": "sha512-cQTmXe3P2t+yDph+nyRtCIapN+dnM+7oIx8ujOqp7LD0SIDzfhN1lbRKAVvVUB47K8CJvmpOQuEZUXVaU/QF2w==", "requires": { "@hibas123/utils": "^2.1.0" } }, "@hibas123/nodelogging": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@hibas123/nodelogging/-/nodelogging-2.1.0.tgz", - "integrity": "sha512-Tms9Y3XQ7e9XV6kbp+fFKXQF9cEE4v6QQ1tzE1e9SKlO//mP/1+TpAmSJ0R+LPBlQR2xVzukvilJMD89xOfFUQ==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@hibas123/nodelogging/-/nodelogging-2.1.1.tgz", + "integrity": "sha512-WiOXmqi2A6EvVSxI9zCISE/aK5TuFeosQvmPbC1BOqIb5Sn6eWsezwGX+BZvUN83aUx/a0dBXT3o+ZcMicORZQ==", "requires": { - "@hibas123/logging": "^2.1.0", - "@hibas123/utils": "^2.1.0" + "@hibas123/logging": "^2.1.1", + "@hibas123/utils": "^2.1.1" } }, "@hibas123/utils": { @@ -81,12 +73,12 @@ } }, "@types/dotenv": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/@types/dotenv/-/dotenv-6.1.1.tgz", - "integrity": "sha512-ftQl3DtBvqHl9L16tpqqzA4YzCSXZfi7g8cQceTz5rOlYtk/IZbFjAv3mLOQlNIgOaylCQWQoBdDQHPgEBJPHg==", + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/@types/dotenv/-/dotenv-8.2.0.tgz", + "integrity": "sha512-ylSC9GhfRH7m1EUXBXofhgx4lUWmFeQDINW5oLuS+gxWdfUeW4zJdeVTYVkexEW+e2VUvlZR2kGnGGipAWR7kw==", "dev": true, "requires": { - "@types/node": "*" + "dotenv": "*" } }, "@types/events": { @@ -95,9 +87,9 @@ "integrity": "sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==" }, "@types/express": { - "version": "4.17.1", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.1.tgz", - "integrity": "sha512-VfH/XCP0QbQk5B5puLqTLEeFgR8lfCJHZJKkInZ9mkYd+u8byX0kztXEQxEk4wZXJs8HI+7km2ALXjn4YKcX9w==", + "version": "4.17.2", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.2.tgz", + "integrity": "sha512-5mHFNyavtLoJmnusB8OKJ5bshSzw+qkMIBAobLrIM48HJvunFva9mOa6aBwh64lBFyNwBbs0xiEFuj4eU/NjCA==", "dev": true, "requires": { "@types/body-parser": "*", @@ -106,9 +98,9 @@ } }, "@types/express-serve-static-core": { - "version": "4.16.9", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.16.9.tgz", - "integrity": "sha512-GqpaVWR0DM8FnRUJYKlWgyARoBUAVfRIeVDZQKOttLFp5SmhhF9YFIYeTPwMd/AXfxlP7xVO2dj1fGu0Q+krKQ==", + "version": "4.16.11", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.16.11.tgz", + "integrity": "sha512-K8d2M5t3tBQimkyaYTXxtHYyoJPUEhy2/omVRnTAKw5FEdT+Ft6lTaTOpoJdHeG+mIwQXXtqiTcYZ6IR8LTzjQ==", "dev": true, "requires": { "@types/node": "*", @@ -137,9 +129,9 @@ "dev": true }, "@types/jsonwebtoken": { - "version": "8.3.4", - "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-8.3.4.tgz", - "integrity": "sha512-R0eiYPpbo6Jl/XtGwpg7vQlapd7D38gp0g9WMKOSFr/e2NUTN9Udd3ty7n+2yUpmHr2Fti+/BSc+KSQ8zPN+vg==", + "version": "8.3.5", + "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-8.3.5.tgz", + "integrity": "sha512-VGM1gb+LwsQ5EPevvbvdnKncajBdYqNcrvixBif1BsiDQiSF1q+j4bBTvKC6Bt9n2kqNSx+yNTY2TVJ360E7EQ==", "dev": true, "requires": { "@types/node": "*" @@ -152,9 +144,9 @@ "dev": true }, "@types/koa": { - "version": "2.0.50", - "resolved": "https://registry.npmjs.org/@types/koa/-/koa-2.0.50.tgz", - "integrity": "sha512-TcgOD2lh0EISSadAk1DOBYw7kNoY9XdeB3vEMOKiDDaTMYm+V54nyPsU7Ulb/htb5OBIR79RgTeCWntCcophLw==", + "version": "2.0.51", + "resolved": "https://registry.npmjs.org/@types/koa/-/koa-2.0.51.tgz", + "integrity": "sha512-L5e/l6Z+SR9Jk6HM0wNYdkvWhSUBOvi+7Q5Uwn7kE/VmBXX7NIxARMigARWAyXAtXiv5Ry1P2HmebolFdvuIVg==", "dev": true, "requires": { "@types/accepts": "*", @@ -184,9 +176,9 @@ } }, "@types/leveldown": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@types/leveldown/-/leveldown-4.0.0.tgz", - "integrity": "sha512-1+h/AaqhhQK1/Q1Nr1sHyy7JDcVmKosL9uYLb4bhdLf7MQP4f4daWRLuFQKp8n/pRbai2XPUpR7z33N85m2Hng==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@types/leveldown/-/leveldown-4.0.1.tgz", + "integrity": "sha512-hrna4yd/zAFkbM/Svic5FPPaPhBiNH4F3hM8eaTtpGI1T5ZT3j1FUO7U/Cd7s93ysEH5J6lP2ZbWGV6SqwimqQ==", "dev": true, "requires": { "@types/abstract-leveldown": "*", @@ -210,9 +202,9 @@ "dev": true }, "@types/node": { - "version": "12.7.12", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.7.12.tgz", - "integrity": "sha512-KPYGmfD0/b1eXurQ59fXD1GBzhSQfz6/lKBxkaHX9dKTzjXbK68Zt7yGUxUsCS1jeTy/8aL+d9JEr+S54mpkWQ==" + "version": "12.12.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.5.tgz", + "integrity": "sha512-KEjODidV4XYUlJBF3XdjSH5FWoMCtO0utnhtdLf1AgeuZLOrRbvmU/gaRCVg7ZaQDjVf3l84egiY0mRNe5xE4A==" }, "@types/range-parser": { "version": "1.2.3", @@ -252,11 +244,12 @@ "dev": true }, "abstract-leveldown": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.1.1.tgz", - "integrity": "sha512-7fK/KySVqzKIomdhkSWzX4YBQhzkzEMbWSiaB6mSN9e+ZdV3KEeKxia/8xQzCkATA5xnnukdP88cFR0D2FsFXw==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.2.tgz", + "integrity": "sha512-/a+Iwj0rn//CX0EJOasNyZJd2o8xur8Ce9C57Sznti/Ilt/cb6Qd8/k98A4ZOklXgTG+iAYYUs1OTG0s1eH+zQ==", "requires": { "level-concat-iterator": "~2.0.0", + "level-supports": "~1.0.0", "xtend": "~4.0.0" } }, @@ -427,6 +420,11 @@ } } }, + "base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + }, "binary-extensions": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz", @@ -495,6 +493,15 @@ } } }, + "buffer": { + "version": "5.4.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.4.3.tgz", + "integrity": "sha512-zvj65TkFeIt3i6aj5bIvJDzjjQQGs4o/sNoezg1F1kYap9Nu2jcUdpwzRSJTHMMzG0H7bZkn4rNQpImhuxWX2A==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, "buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -679,9 +686,9 @@ "dev": true }, "commander": { - "version": "2.20.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz", - "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==", + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "optional": true }, "component-emitter": { @@ -741,12 +748,19 @@ "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" }, "cookies": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/cookies/-/cookies-0.7.3.tgz", - "integrity": "sha512-+gixgxYSgQLTaTIilDHAdlNPZDENDQernEMiIcZpYYP14zgHsCt4Ce1FEjFtcp6GefhozebB6orvhAAWx/IS0A==", + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/cookies/-/cookies-0.8.0.tgz", + "integrity": "sha512-8aPsApQfebXnuI+537McwYsDtjVxGm8gTIzQI3FDW6t5t/DAhERxtnbEPN/8RX+uZthoz4eCOgloXaE5cYyNow==", "requires": { - "depd": "~1.1.2", - "keygrip": "~1.0.3" + "depd": "~2.0.0", + "keygrip": "~1.1.0" + }, + "dependencies": { + "depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" + } } }, "copy-descriptor": { @@ -920,9 +934,9 @@ } }, "dotenv": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.1.0.tgz", - "integrity": "sha512-GUE3gqcDCaMltj2++g6bRQ5rBJWtkWTmqmD0fo1RnnMuUqHNCt2oTPeDnS9n6fKYvlhn7AeBkb38lymBtWBQdA==" + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", + "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==" }, "duplexer3": { "version": "0.1.4", @@ -1819,15 +1833,15 @@ } }, "graceful-fs": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.2.tgz", - "integrity": "sha512-IItsdsea19BoLC7ELy13q1iJFNmd7ofZH5+X/pJr90/nRoPEX0DJo1dHDbgtYWOhJhcCgMDTOw84RZ72q6lB+Q==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz", + "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==", "dev": true }, "handlebars": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.4.3.tgz", - "integrity": "sha512-B0W4A2U1ww3q7VVthTKfh+epHx+q4mCt6iK+zEAzbMBpWQAwxCeKxEGpj/1oQTpzPXDNSOG7hmG14TsISH50yw==", + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.5.1.tgz", + "integrity": "sha512-C29UoFzHe9yM61lOsIlCE5/mQVGrnIOrOq7maQl76L7tYPCgC1og0Ajt6uWnX4ZTxBPnjw+CUvawphwCfJgUnA==", "requires": { "neo-async": "^2.6.0", "optimist": "^0.6.1", @@ -1915,6 +1929,11 @@ "safer-buffer": ">= 2.1.2 < 3" } }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "ignore-by-default": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", @@ -1946,7 +1965,8 @@ "ini": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "dev": true }, "invert-kv": { "version": "2.0.0", @@ -2227,9 +2247,12 @@ } }, "keygrip": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.0.3.tgz", - "integrity": "sha512-/PpesirAIfaklxUzp4Yb7xBper9MwP6hNRA6BGGUFCgbJ+BM5CKBtsoxinNXkLHAr+GXS1/lSlF2rP7cv5Fl+g==" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz", + "integrity": "sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==", + "requires": { + "tsscmp": "1.0.6" + } }, "kind-of": { "version": "6.0.2", @@ -2238,15 +2261,15 @@ "dev": true }, "koa": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/koa/-/koa-2.8.2.tgz", - "integrity": "sha512-q1uZOgpl3wjr5FS/tjbABJ8lA5+NeKa9eq7QyBP5xxgOBwJN4iBrMEgO3LroE51lrIw3BsO0WZZ0Yi6giSiMDw==", + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/koa/-/koa-2.11.0.tgz", + "integrity": "sha512-EpR9dElBTDlaDgyhDMiLkXrPwp6ZqgAIBvhhmxQ9XN4TFgW+gEz6tkcsNI6BnUbUftrKDjVFj4lW2/J2aNBMMA==", "requires": { "accepts": "^1.3.5", "cache-content-type": "^1.0.0", "content-disposition": "~0.5.2", "content-type": "^1.0.4", - "cookies": "~0.7.1", + "cookies": "~0.8.0", "debug": "~3.1.0", "delegates": "^1.0.0", "depd": "^1.1.2", @@ -2260,7 +2283,6 @@ "is-generator-function": "^1.0.7", "koa-compose": "^4.1.0", "koa-convert": "^1.2.0", - "koa-is-json": "^1.0.0", "on-finished": "^2.3.0", "only": "~0.0.2", "parseurl": "^1.3.2", @@ -2313,11 +2335,6 @@ } } }, - "koa-is-json": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/koa-is-json/-/koa-is-json-1.0.0.tgz", - "integrity": "sha1-JzwH7c3Ljfaiwat9We52SRRR7BQ=" - }, "koa-router": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/koa-router/-/koa-router-7.4.0.tgz", @@ -2403,11 +2420,11 @@ } }, "leveldown": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-5.3.0.tgz", - "integrity": "sha512-PQXwTKMz55rYlg7984VbM7xpcqdiWgVKRms2fEgqVL7spd6+wK8NewScJOYIGpIG7/XxMOc0i+q/NX0WLJEcwA==", + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-5.4.1.tgz", + "integrity": "sha512-3lMPc7eU3yj5g+qF1qlALInzIYnkySIosR1AsUKFjL9D8fYbTLuENBAeDRZXIG4qeWOAyqRItOoLu2v2avWiMA==", "requires": { - "abstract-leveldown": "~6.1.1", + "abstract-leveldown": "~6.2.1", "napi-macros": "~2.0.0", "node-gyp-build": "~4.1.0" } @@ -2684,18 +2701,18 @@ "integrity": "sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ==" }, "nodemon": { - "version": "1.19.3", - "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-1.19.3.tgz", - "integrity": "sha512-TBNKRmJykEbxpTniZBusqRrUTHIEqa2fpecbTQDQj1Gxjth7kKAPP296ztR0o5gPUWsiYbuEbt73/+XMYab1+w==", + "version": "1.19.4", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-1.19.4.tgz", + "integrity": "sha512-VGPaqQBNk193lrJFotBU8nvWZPqEZY2eIzymy2jjY0fJ9qIsxA0sxQ8ATPl0gZC645gijYEc1jtZvpS8QWzJGQ==", "dev": true, "requires": { - "chokidar": "^2.1.5", - "debug": "^3.1.0", + "chokidar": "^2.1.8", + "debug": "^3.2.6", "ignore-by-default": "^1.0.1", "minimatch": "^3.0.4", - "pstree.remy": "^1.1.6", - "semver": "^5.5.0", - "supports-color": "^5.2.0", + "pstree.remy": "^1.1.7", + "semver": "^5.7.1", + "supports-color": "^5.5.0", "touch": "^3.1.0", "undefsafe": "^2.0.2", "update-notifier": "^2.5.0" @@ -2996,6 +3013,11 @@ "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=", "dev": true }, + "pretty-bytes": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.3.0.tgz", + "integrity": "sha512-hjGrh+P926p4R4WbaB6OckyRtO0F0/lQBiT+0gnxjV+5kjPBrfVBFCsCLbMqVQeydvIoouYTCmmEURiH3R1Bdg==" + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -3666,6 +3688,11 @@ "integrity": "sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==", "dev": true }, + "tsscmp": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/tsscmp/-/tsscmp-1.0.6.tgz", + "integrity": "sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==" + }, "type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -3682,12 +3709,12 @@ "dev": true }, "uglify-js": { - "version": "3.6.1", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.1.tgz", - "integrity": "sha512-+dSJLJpXBb6oMHP+Yvw8hUgElz4gLTh82XuX68QiJVTXaE5ibl6buzhNkQdYhBlIhozWOC9ge16wyRmjG4TwVQ==", + "version": "3.6.7", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.7.tgz", + "integrity": "sha512-4sXQDzmdnoXiO+xvmTzQsfIiwrjUCSA95rSP4SEd8tDb51W2TiDOlL76Hl+Kw0Ie42PSItCW8/t6pBNCF2R48A==", "optional": true, "requires": { - "commander": "2.20.0", + "commander": "~2.20.3", "source-map": "~0.6.1" }, "dependencies": { @@ -3861,6 +3888,15 @@ "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" }, + "what-the-pack": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/what-the-pack/-/what-the-pack-2.0.3.tgz", + "integrity": "sha512-vJFrS6U6acWUgkKu9dLuMsQnnHEzIpop1VDMc142h+zU+jUcS+FZlRO/BDJ9/S9cvp2sQWygImBDwl0WN9aadw==", + "requires": { + "buffer": "^5.2.1", + "pretty-bytes": "^5.1.0" + } + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", @@ -3955,9 +3991,9 @@ } }, "ws": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.1.2.tgz", - "integrity": "sha512-gftXq3XI81cJCgkUiAVixA0raD9IVmXqsylCrjRygw4+UOOGzPoxnQ6r/CnVL9i+mDncJo94tSkyrtuuQVBmrg==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.2.0.tgz", + "integrity": "sha512-+SqNqFbwTm/0DC18KYzIsMTnEWpLwJsiasW/O17la4iDRRIO9uaHbvKiAS3AHgTiuuWerK/brj4O6MYZkei9xg==", "requires": { "async-limiter": "^1.0.0" } diff --git a/package.json b/package.json index 054a1b6..7e3e8f3 100644 --- a/package.json +++ b/package.json @@ -15,34 +15,33 @@ "author": "Fabian Stamm ", "license": "ISC", "devDependencies": { - "@types/dotenv": "^6.1.1", - "@types/ini": "^1.3.30", - "@types/jsonwebtoken": "^8.3.4", - "@types/koa": "^2.0.50", + "@types/dotenv": "^8.2.0", + "@types/jsonwebtoken": "^8.3.5", + "@types/koa": "^2.0.51", "@types/koa-router": "^7.0.42", - "@types/leveldown": "^4.0.0", + "@types/leveldown": "^4.0.1", "@types/levelup": "^3.1.1", - "@types/node": "^12.7.12", + "@types/node": "^12.12.5", "@types/shortid": "0.0.29", "@types/ws": "^6.0.3", "concurrently": "^5.0.0", - "nodemon": "^1.19.3", + "nodemon": "^1.19.4", "typescript": "^3.6.4" }, "dependencies": { - "@hibas123/binary-encoder": "^1.0.0", - "@hibas123/nodelogging": "^2.1.0", + "@hibas123/logging": "^2.1.1", + "@hibas123/nodelogging": "^2.1.1", "@hibas123/utils": "^2.1.1", - "dotenv": "^8.1.0", - "handlebars": "^4.4.3", - "ini": "^1.3.5", + "dotenv": "^8.2.0", + "handlebars": "^4.5.1", "jsonwebtoken": "^8.5.1", - "koa": "^2.8.2", + "koa": "^2.11.0", "koa-body": "^4.1.1", "koa-router": "^7.4.0", - "leveldown": "^5.3.0", + "leveldown": "^5.4.1", "levelup": "^4.3.2", "shortid": "^2.2.15", - "ws": "^7.1.2" + "what-the-pack": "^2.0.3", + "ws": "^7.2.0" } -} +} \ No newline at end of file diff --git a/src/connection.ts b/src/connection.ts index 19786ef..ce30c2c 100644 --- a/src/connection.ts +++ b/src/connection.ts @@ -21,7 +21,7 @@ async function verifyJWT(token: string, publicKey: string) { import { URLSearchParams } from "url"; -type QueryTypes = "keys" | "get" | "set" | "push" | "subscribe" | "unsubscribe"; +type QueryTypes = "keys" | "get" | "set" | "update" | "delete" | "push" | "subscribe" | "unsubscribe"; export class ConnectionManager { static server: WebSocket.Server; @@ -36,7 +36,7 @@ export class ConnectionManager { const sendError = (error: string) => socket.send(JSON.stringify({ ns: "error_msg", data: error })); - const session = new Session(); + const session = new Session(shortid()); let query = new URLSearchParams(req.url.split("?").pop()); @@ -83,7 +83,7 @@ export class ConnectionManager { } const handler = new Map void)>(); - type QueryData = { id: string, type: QueryTypes, path: string[], data: any }; + type QueryData = { id: string, type: QueryTypes, path: string[], data: any, options: any }; handler.set("query", async ({ id, type, path, data }: QueryData) => { //TODO: Handle case with no id, type, path Logging.debug(`Request with id '${id}' from type '${type}' and path '${path}' with data`, data) @@ -93,7 +93,7 @@ export class ConnectionManager { if (!db) answer(id, "Database not found!", true); else { - const query = stored.get(id) || db.getQuery(path); + const query = stored.get(id) || db.getQuery(path, session.sessionid); switch (type) { case "keys": @@ -109,7 +109,17 @@ export class ConnectionManager { case "set": if (!perms.write) throw noperm; - answer(id, await query.set(data)); + answer(id, await query.set(data, {})); + return; + case "update": + if (!perms.write) + throw noperm; + answer(id, await query.update(data)); + return; + case "delete": + if (!perms.write) + throw noperm; + answer(id, await query.delete()); return; case "push": if (!perms.write) @@ -122,9 +132,9 @@ export class ConnectionManager { let subscriptionID = shortid.generate(); - query.subscribe(data, (data) => { + query.subscribe((data) => { socket.send(JSON.stringify({ ns: "event", data: { id: subscriptionID, data } })); - }); + }, data.types, data.options); stored.set(id, query); answer(id, subscriptionID); return; diff --git a/src/database/database.ts b/src/database/database.ts index bbb339a..cf44152 100644 --- a/src/database/database.ts +++ b/src/database/database.ts @@ -1,9 +1,8 @@ import { Rules } from "./rules"; import Settings from "../settings"; -import getLevelDB from "../storage"; -import PathLock from "./lock"; +import getLevelDB, { LevelDB, deleteLevelDB } from "../storage"; +import DocumentLock from "./lock"; import Query from "./query"; -import { Observable } from "@hibas123/utils"; export class DatabaseManager { static databases = new Map(); @@ -36,27 +35,33 @@ export class DatabaseManager { if (db) { await Settings.deleteDatabase(name); await db.stop(); - + await deleteLevelDB(db.name) } } } -export enum ChangeTypes { - SET, - PUSH -} +export type ChangeTypes = "added" | "modified" | "deleted"; export type Change = { + data: any; + document: string; type: ChangeTypes; - path: string[] + sender: string; } -export class Database { - public level = getLevelDB(this.name); - public rules: Rules; - public locks = new PathLock() - public changeObservable = new Observable(); +export class Database { + private level = getLevelDB(this.name); + + get data() { + return this.level; + } + + + public rules: Rules; + public locks = new DocumentLock() + + public changes = new Map void>>(); toJSON() { return { @@ -94,11 +99,11 @@ export class Database { } - getQuery(path: string[]) { - return new Query(this, path); + getQuery(path: string[], sender: string) { + return new Query(this, path, sender); } async stop() { - await this.level.close(); + await this.data.close(); } } diff --git a/src/database/lock.ts b/src/database/lock.ts index 06ca56e..dc1e825 100644 --- a/src/database/lock.ts +++ b/src/database/lock.ts @@ -1,43 +1,23 @@ export type Release = { release: () => void }; -export default class PathLock { - locks: { - path: string[], - next: (() => void)[] - }[] = []; +export default class DocumentLock { + private locks = new Map void)[]>(); - constructor() { } - - async lock(path: string[]) { - let locks = this.locks.filter(lock => { - let idxs = Math.min(lock.path.length, path.length); - if (idxs === 0) return true; - for (let i = 0; i < idxs; i++) { - if (lock.path[i] !== path[i]) - return false; - } - return true; - }) - - if (locks.length > 0) { // await till release - await Promise.all(locks.map(l => new Promise(res => l.next.push(res)))) - } else { - let lock = { - path: path, - next: [] - } - this.locks.push(lock); - locks = [lock]; + async lock(collection: string = "", document: string = "") { + let key = collection + "/" + document; + let l = this.locks.get(key); + if (l) + await new Promise(resolve => { l.push(resolve); this.locks.set(key, l) }); + else { + l = []; + this.locks.set(key, l); } return () => { - locks.forEach(lock => { - if (lock.next.length > 0) { - setImmediate(() => lock.next.shift()()); - } else { - this.locks.splice(this.locks.indexOf(lock), 1); - } - }) + if (l.length > 0) + setImmediate(() => l.shift()); + else + this.locks.delete(key) } } } \ No newline at end of file diff --git a/src/database/query.ts b/src/database/query.ts index ef26e27..fb32ed5 100644 --- a/src/database/query.ts +++ b/src/database/query.ts @@ -1,275 +1,265 @@ import { Database, Change, ChangeTypes } from "./database"; - -import Encoder, { DataTypes } from "@hibas123/binary-encoder"; -import { resNull } from "../storage"; +import { resNull, LevelDB } from "../storage"; import { LevelUpChain } from "levelup"; import shortid = require("shortid"); import Logging from "@hibas123/nodelogging"; +import * as MSGPack from "what-the-pack"; -enum FieldTypes { - OBJECT, - VALUE +export const MP = MSGPack.initialize(2 ** 20); + +const { encode, decode } = MP; + +interface ISubscribeOptions { + existing: boolean; } -interface IField { - type: FieldTypes; - value?: any -} - -export const FieldEncoder = new Encoder({ - type: { - index: 1, - type: DataTypes.UINT8 - }, - value: { - index: 3, - type: DataTypes.AUTO, - allowNull: true - } -}) - export default class Query { - constructor(private database: Database, private path: string[]) { + /** + * Returns true if the path only contains valid characters and false if it doesn't + * @param path Path to be checked + */ + private validatePath(path: string[]) { + return path.every(e => (e.match(/[^a-zA-Z0-9_\-\<\>]/g) || []).length === 0); + } + + constructor(private database: Database, private path: string[], private sender: string) { if (path.length > 10) { throw new Error("Path is to long. Path is only allowed to be 10 Layers deep!"); } - if (path.find(segment => segment.indexOf("/") >= 0)) { - throw new Error("Path cannot contain '/'!"); + if (!this.validatePath(path)) { + throw new Error("Path can only contain a-z A-Z 0-9 '-' '-' '<' and '>' "); } this.onChange = this.onChange.bind(this); } - private pathToKey(path?: string[]) { - return "/" + (path || this.path).join("/"); + + private async resolve(path: string[], create = false): Promise<{ collection: string, document: string }> { + path = [...path]; // Create modifiable copy + let collectionID: string = undefined; + let documentKey = undefined; + while (path.length > 0) { + let collectionName = path.shift(); + let key = `${collectionID || ""}/${documentKey || ""}/${collectionName}`; + let lock = await this.database.locks.lock(collectionID, documentKey); + try { + collectionID = await this.database.data.get(key).then(r => r.toString()).catch(resNull); + + if (!collectionID) { + if (create) { + collectionID = shortid.generate(); + await this.database.data.put(key, Buffer.from(collectionID)); + } else { + return { collection: null, document: null }; + } + } + + + if (path.length > 0) + documentKey = path.shift(); + else + documentKey = undefined; + } finally { + lock(); + } + } + + return { + collection: collectionID, + document: documentKey + }; } - private getField(path: string[]): Promise { - return this.database.level.get(this.pathToKey(path), { asBuffer: true }).then((res: Buffer) => FieldEncoder.decode(res)).catch(resNull); + private getKey(collection: string, document?: string) { + return `${collection || ""}/${document || ""}`; } - private getFields(path: string[]) { - let p = this.pathToKey(path); - if (!p.endsWith("/")) - p += "/"; - let t = Buffer.from(p); + private getDoc(collection: string, document: string) { + return this.database.data.get(this.getKey(collection, document), { asBuffer: true }).then(res => decode(res as Buffer)).catch(resNull); + } - let gt = Buffer.alloc(t.length + 1); - gt.set(t); - gt[t.length] = 0; + private sendChange(collection: string, document: string, type: ChangeTypes, data: any) { + let change: Change = { + type, + document, + data, + sender: this.sender + } - let lt = Buffer.alloc(t.length + 1); - lt.set(t); - lt[t.length] = 0xFF; + let s = this.database.changes.get(this.getKey(collection, document)) + if (s) + s.forEach(e => setImmediate(() => e(change))) + s = this.database.changes.get(this.getKey(collection)) + if (s) + s.forEach(e => setImmediate(() => e(change))) + + } + + public async get() { + let { collection, document } = await this.resolve(this.path); + + if (!collection || !document) { + return null; + } + const lock = await this.database.locks.lock(collection, document); + return this.getDoc(collection, document).finally(() => lock()) + } + + public async keys() { + let { collection, document } = await this.resolve(this.path); + if (document) + throw new Error("Keys only works on collections!"); + if (!collection) + throw new Error("There must be a collection"); + + + let gt = Buffer.from(this.getKey(collection) + " "); + gt[gt.length - 1] = 0; + + let lt = Buffer.alloc(gt.length); + lt.set(gt); + lt[gt.length - 1] = 0xFF; return new Promise((yes, no) => { let keys = []; - const stream = this.database.level.createKeyStream({ - gt: Buffer.from(p), - lt: Buffer.from(lt) + const stream = this.database.data.createKeyStream({ + gt, + lt }) - stream.on("data", key => keys.push(key.toString())); + stream.on("data", (key: string | Buffer) => { + key = key.toString(); + let s = key.split("/"); + if (s.length === 2) + keys.push(s[1]); + }); stream.on("end", () => yes(keys)); stream.on("error", no); }); } - async keys() { - const lock = await this.database.locks.lock(this.path); - try { - let obj = await this.getField(this.path); - if (!obj) - return []; - let fields = await this.getFields(this.path); - return fields.map(field => field.split("/").filter(e => e !== "")).filter(path => path.length === this.path.length + 1).map(path => path.pop()); - } finally { - lock() + public async set(data: any, { merge = false }) { + if (data === null) + return this.delete(); + let { collection, document } = await this.resolve(this.path, true); + if (!collection) { + throw new Error("There must be a collection!") } + + if (!document) { + throw new Error("There must be a document key!") + } + + const lock = await this.database.locks.lock(collection, document); + + let isNew = await this.database.data.get(this.getKey(collection, document)).catch(resNull).then(e => e !== null); + + return this.database.data + .put(this.getKey(collection, document), encode(data)) + .then(() => this.sendChange(collection, document, isNew ? "added" : "modified", data)) + .finally(() => lock()) } - async get() { - const lock = await this.database.locks.lock(this.path); - try { - const getData = async (path: string[]) => { - let obj = await this.getField(path); - if (!obj) - return null; - else { - if (obj.type === FieldTypes.VALUE) { - return obj.value; - } else { - let fields = await this.getFields(this.path); - let sorted = fields.map(field => field.split("/").filter(e => e !== "")).sort((a, b) => a.length - b.length) - - let res = {}; - for (let path of sorted) { - let field = await this.getField(path); - let shortened = path.slice(this.path.length); - let t = res; - for (let section of shortened.slice(0, -1)) { - t = t[section]; - } - - if (field.type === FieldTypes.OBJECT) { - t[path[path.length - 1]] = {}; - } else { - t[path[path.length - 1]] = field.value; - } - } - - return res; - } - } - } - return await getData(this.path); - } finally { - lock(); - } - } - - async push(value: any) { + public async push(value: any) { let id = shortid.generate(); - let q = new Query(this.database, [...this.path, id]); - await q.set(value); - this.database.changeObservable.send({ - path: [...this.path, id], - type: ChangeTypes.PUSH - }) + let q = new Query(this.database, [...this.path, id], this.sender); + await q.set(value, {}); return id; } - async set(value: any) { - const lock = await this.database.locks.lock(this.path); - let batch = this.database.level.batch(); - try { - if (value === null || value === undefined) { - this.delete(value); - } else { - let field = await this.getField(this.path); - if (field) { - await this.delete(batch); - } else { - for (let i = 0; i < this.path.length; i++) { - let subpath = this.path.slice(0, i); - let field = await this.getField(subpath); - if (!field) { - batch.put(this.pathToKey(subpath), FieldEncoder.encode({ - type: FieldTypes.OBJECT - })); - } else if (field.type !== FieldTypes.OBJECT) { - throw new Error("Parent elements not all Object. Cannot set value!"); - } - } - } + public async delete() { + let { collection, document } = await this.resolve(this.path); - const saveValue = (path: string[], value: any) => { - if (typeof value === "object") { - //TODO: Handle case array! - // Field type array? - batch.put(this.pathToKey(path), FieldEncoder.encode({ - type: FieldTypes.OBJECT - })) - for (let field in value) { - saveValue([...path, field], value[field]); - } - } else { - batch.put(this.pathToKey(path), FieldEncoder.encode({ - type: FieldTypes.VALUE, - value - })); - } - } - - saveValue(this.path, value); - } - - await batch.write(); - this.database.changeObservable.send({ - path: this.path, - type: ChangeTypes.SET - }) - } catch (err) { - if (batch.length > 0) - batch.clear(); - throw err; - } finally { - lock(); + if (!collection) { + throw new Error("There must be a collection!") } + + if (!document) { + throw new Error("There must be a document key!") + } + + + const lock = await this.database.locks.lock(collection, document); + + return await this.database.data + .del(`${collection}/${document}`) + .then(() => this.sendChange(collection, document, "deleted", null)) + .finally(() => lock()) } - private async delete(batch?: LevelUpChain) { - let lock = batch ? undefined : await this.database.locks.lock(this.path); - const commit = batch ? false : true; - if (!batch) - batch = this.database.level.batch(); - try { - let field = await this.getField(this.path); - if (field) { - let fields = await this.getFields(this.path) - fields.forEach(field => batch.del(field)); - batch.del(this.pathToKey(this.path)); - } - - if (commit) - await batch.write(); - } catch (err) { - if (batch.length > 0) - batch.clear() - } finally { - if (lock) - lock() - } + public async update(data: any) { + //TODO: Implement } - subscription: { - type: ChangeTypes; - send: (data: any) => void; + key: string; + type: Set; + send: (data: Omit) => void; }; - subscribe(type: "set" | "push", send: (data: any) => void) { + async subscribe(send: (data: Omit) => void, type: ChangeTypes[] | undefined, _options: Partial) { + let options: ISubscribeOptions = { + existing: true, + ..._options + } + + let { collection, document } = await this.resolve(this.path); + + let key = this.getKey(collection, document); + let s = this.database.changes.get(key) || new Set(); + s.add(this.onChange) + this.database.changes.set(key, s); + this.subscription = { + key, send, - type: type === "set" ? ChangeTypes.SET : ChangeTypes.PUSH + type: new Set(type || []) }; - this.database.changeObservable.subscribe(this.onChange); - Logging.debug("Subscribe"); + + if (options.existing) { + if (document) { + send({ + document: document, + type: "added", + data: await this.get(), + }) + } else { + await this.keys().then(async documents => { + for (let document of documents) { + const data = await this.getDoc(collection, document); + if (data) + send({ + type: "added", + document, + data + }) + } + }) + } + } + + Logging.debug("Subscribed"); } async onChange(change: Change) { - Logging.debug("Change:", change); - if (!this.subscription) - return this.database.changeObservable.unsubscribe(this.onChange); - const { type, send } = this.subscription; - if (type === change.type) { - Logging.debug("Path", this.path, change.path); - if (this.path.length <= change.path.length - (type === ChangeTypes.PUSH ? 1 : 0)) { - let valid = true; - for (let i = 0; i < this.path.length; i++) { - if (this.path[i] !== change.path[i]) { - valid = false; - break; - } - } - if (valid) { - Logging.debug("Send Change:", change); - if (type === ChangeTypes.PUSH) { - send({ - id: change.path[change.path.length - 1], - path: change.path, - data: await new Query(this.database, change.path).get() - }) - } else { - send(await this.get()) - } - } - } + // Events from the sender are handled locally + if (change.sender !== this.sender && type.has(change.type)) { + let c = { ...change }; + delete c.sender; + send(c) } } unsubscribe() { - this.subscription = undefined; - this.database.changeObservable.unsubscribe(this.onChange); + if (!this.subscription) { + const { key } = this.subscription; + let s = this.database.changes.get(key); + if (s) { + s.delete(this.onChange); + if (s.size <= 0) + this.database.changes.delete(key); + } + this.subscription = undefined; + } } } \ No newline at end of file diff --git a/src/database/session.ts b/src/database/session.ts index 4f11346..bc6811c 100644 --- a/src/database/session.ts +++ b/src/database/session.ts @@ -1,4 +1,8 @@ export default class Session { + constructor(private _sessionid: string) { } + get sessionid() { + return this._sessionid; + } root: boolean = false; uid: string = undefined; } \ No newline at end of file diff --git a/src/storage.ts b/src/storage.ts index 3739099..f4dbc86 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -12,26 +12,27 @@ export type LevelDB = LU>; const databases = new Map(); -export function resNull(err) { +export function resNull(err): null { if (!err.notFound) throw err; return null; } -function rmRecursice(path: string) { +async function rmRecursice(path: string) { if (fs.existsSync(path)) { - fs.readdirSync(path).forEach(function (file, index) { + await Promise.all(fs.readdirSync(path).map(async (file) => { var curPath = path + "/" + file; if (fs.lstatSync(curPath).isDirectory()) { // recurse - rmRecursice(curPath); + await rmRecursice(curPath); } else { // delete file - fs.unlinkSync(curPath); + await fs.promises.unlink(curPath); } - }); - fs.rmdirSync(path); + })); + await fs.promises.rmdir(path); } }; + export async function deleteLevelDB(name: string) { if (!name || name === "") return; @@ -42,7 +43,7 @@ export async function deleteLevelDB(name: string) { //TODO make sure, that name doesn't make it possible to delete all databases :) - rmRecursice("./databases/" + name); + await rmRecursice("./databases/" + name); } export default function getLevelDB(name: string): LevelDB { diff --git a/src/types.d.ts b/src/types.d.ts new file mode 100644 index 0000000..bf8172f --- /dev/null +++ b/src/types.d.ts @@ -0,0 +1,10 @@ +declare module "what-the-pack" { + namespace whatthepack { + function initialize(bufferSize: number): { + encode(data: any): Buffer; + decode(data: Buffer): T; + Buffer: typeof global.Buffer + } + } + export = whatthepack; +} \ No newline at end of file diff --git a/src/web/v1/admin.ts b/src/web/v1/admin.ts index cec956b..8bd4c23 100644 --- a/src/web/v1/admin.ts +++ b/src/web/v1/admin.ts @@ -4,8 +4,9 @@ import getForm from "../helper/form"; import getTable from "../helper/table"; import { BadRequestError, NoPermissionError } from "../helper/errors"; import { DatabaseManager } from "../../database/database"; -import { FieldEncoder } from "../../database/query"; +import { MP } from "../../database/query"; import config from "../../config"; +import Logging from "@hibas123/logging"; const AdminRoute = new Router(); @@ -46,14 +47,16 @@ AdminRoute.get("/data", async ctx => { throw new BadRequestError("Database not found"); let res = await new Promise((yes, no) => { - const stream = db.level.createReadStream({ + const stream = db.data.createReadStream({ keys: true, values: true, - valueAsBuffer: true + valueAsBuffer: true, + keyAsBuffer: false }); let res = [["key", "value"]]; - stream.on("data", ({ key, value }) => { - res.push([key, JSON.stringify(FieldEncoder.decode(value))]); + stream.on("data", ({ key, value }: { key: string, value: Buffer }) => { + Logging.debug("Admin Key:", key); + res.push([key, key.split("/").length > 2 ? value.toString() : JSON.stringify(MP.decode(value))]); }) stream.on("error", no);