From 6fc258cf3abdf13b6122ea9f6e452d011adb8a86 Mon Sep 17 00:00:00 2001
From: Fabian Stamm <dev@fabianstamm.de>
Date: Tue, 28 Jul 2020 14:39:54 +0200
Subject: [PATCH] First Commit of registry itself

---
 .drone.status            |   3 +
 .drone.yml               |  17 +++++
 .editorconfig            |   2 +
 registry/.gitignore      |   1 +
 registry/Dockerfile      |  33 ++++++++++
 registry/data/config.ini |  11 ++++
 registry/data/db.json    |   1 +
 registry/dev.sh          |   5 ++
 registry/import_map.json |   5 ++
 registry/src/config.ts   |  10 +++
 registry/src/db.ts       |  13 ++++
 registry/src/deps.ts     |  18 ++++++
 registry/src/http.ts     |  18 ++++++
 registry/src/http/api.ts | 133 +++++++++++++++++++++++++++++++++++++++
 registry/src/http/raw.ts |  53 ++++++++++++++++
 registry/src/registry.ts |  18 ++++++
 registry/src/s3.ts       |  16 +++++
 registry/src/utils.ts    |  86 +++++++++++++++++++++++++
 18 files changed, 443 insertions(+)
 create mode 100644 .drone.status
 create mode 100644 .drone.yml
 create mode 100644 registry/.gitignore
 create mode 100644 registry/Dockerfile
 create mode 100644 registry/data/config.ini
 create mode 100644 registry/data/db.json
 create mode 100755 registry/dev.sh
 create mode 100644 registry/import_map.json
 create mode 100644 registry/src/config.ts
 create mode 100644 registry/src/db.ts
 create mode 100644 registry/src/deps.ts
 create mode 100644 registry/src/http.ts
 create mode 100644 registry/src/http/api.ts
 create mode 100644 registry/src/http/raw.ts
 create mode 100644 registry/src/registry.ts
 create mode 100644 registry/src/s3.ts
 create mode 100644 registry/src/utils.ts

diff --git a/.drone.status b/.drone.status
new file mode 100644
index 0000000..437aa9c
--- /dev/null
+++ b/.drone.status
@@ -0,0 +1,3 @@
+{
+   "url": "https://drone.hibas123.de/Deno/DenReg/"
+}
diff --git a/.drone.yml b/.drone.yml
new file mode 100644
index 0000000..cbcbb69
--- /dev/null
+++ b/.drone.yml
@@ -0,0 +1,17 @@
+kind: pipeline
+type: docker
+name: default
+
+steps:
+  - name: Build denreg registry docker image
+    image: plugins/docker
+    settings:
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      auto_tag: true
+      repo: hibas123.azurecr.io/denreg
+      registry: hibas123.azurecr.io
+      dockerfile: registry/Dockerfile
+      debug: true
diff --git a/.editorconfig b/.editorconfig
index e5c48e1..e8f6ff3 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -5,3 +5,5 @@ end_of_line = lf
 indent_size = 3
 indent_style = space
 insert_final_newline = true
+[*.yml]
+indent_size = 2
diff --git a/registry/.gitignore b/registry/.gitignore
new file mode 100644
index 0000000..3fec32c
--- /dev/null
+++ b/registry/.gitignore
@@ -0,0 +1 @@
+tmp/
diff --git a/registry/Dockerfile b/registry/Dockerfile
new file mode 100644
index 0000000..b17c172
--- /dev/null
+++ b/registry/Dockerfile
@@ -0,0 +1,33 @@
+FROM debian:bullseye-slim AS builder
+
+ENV DENO_VERSION=1.2.1
+
+WORKDIR /build
+
+RUN apt-get update && apt-get install -y curl zip
+
+RUN curl -fsSL https://github.com/denoland/deno/releases/download/v${DENO_VERSION}/deno-x86_64-unknown-linux-gnu.zip --output deno.zip
+RUN unzip deno.zip
+RUN rm deno.zip
+RUN chmod 777 deno
+RUN chmod +x deno
+RUN mv deno /usr/bin/deno
+
+FROM debian:bullseye-slim
+
+COPY --from=builder /usr/bin/deno /usr/bin/deno
+
+RUN ls /usr/bin/deno
+
+WORKDIR /app
+
+COPY src/deps.ts /app/src/deps.ts
+
+RUN /usr/bin/deno cache --unstable src/deps.ts
+
+ADD src /app/src
+
+RUN /usr/bin/deno cache --unstable src/registry.ts
+
+VOLUME [ "/data" ]
+ENTRYPOINT [ "/usr/bin/deno", "run", "-A", "--unstable", "/app/src/registry.ts" ]
diff --git a/registry/data/config.ini b/registry/data/config.ini
new file mode 100644
index 0000000..6a50e8d
--- /dev/null
+++ b/registry/data/config.ini
@@ -0,0 +1,11 @@
+[s3]
+endpoint=https://minio.hibas123.de
+bucket=deno-registry
+accessKey=h6MlrWqvhR1V2id1Q5Ei
+secretKey=zQaJkX9fXvouCVtnH0cSgJyJojsrIiCQPrsQfdnI
+
+[user.hibas123]
+password=asd
+
+[user.asd]
+password=asd
diff --git a/registry/data/db.json b/registry/data/db.json
new file mode 100644
index 0000000..7f1f5e3
--- /dev/null
+++ b/registry/data/db.json
@@ -0,0 +1 @@
+{"x":[{"name":"test","versions":["0.0.1"],"_id":"7ee9bbb0-ce89-11ea-bfc4-7fc1af1f206a"},{"name":"@denreg-cli","versions":["0.0.1","0.0.2","1.2.5","1.3.5"],"_id":"5e096590-cf32-11ea-905e-2929f303405b"}]}
\ No newline at end of file
diff --git a/registry/dev.sh b/registry/dev.sh
new file mode 100755
index 0000000..cffc3a5
--- /dev/null
+++ b/registry/dev.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+deno run -A https://raw.githubusercontent.com/hibas123/denovamon/master/denovamon.ts start --ignore="db.json,tmp" --command="deno run -A --unstable --importmap import_map.json src/registry.ts"
+
+# deno run -A --unstable --importmap import_map.json registry.ts
diff --git a/registry/import_map.json b/registry/import_map.json
new file mode 100644
index 0000000..6c4d67f
--- /dev/null
+++ b/registry/import_map.json
@@ -0,0 +1,5 @@
+{
+   "imports": {
+      "https://deno.land/std@0.60.0/": "https://deno.land/std@0.62.0/"
+   }
+}
diff --git a/registry/src/config.ts b/registry/src/config.ts
new file mode 100644
index 0000000..9a9c8c2
--- /dev/null
+++ b/registry/src/config.ts
@@ -0,0 +1,10 @@
+import { Ini } from "./deps.ts";
+
+const config =
+   Ini.decode(
+      await Deno.readFile("./data/config.ini").then((e) =>
+         new TextDecoder().decode(e)
+      )
+   ) || {};
+
+export default config;
diff --git a/registry/src/db.ts b/registry/src/db.ts
new file mode 100644
index 0000000..df0358d
--- /dev/null
+++ b/registry/src/db.ts
@@ -0,0 +1,13 @@
+import { Datastore } from "./deps.ts";
+
+export interface IPackage {
+   name: string;
+   versions: string[];
+}
+
+const db = new Datastore<IPackage>({
+   filename: "data/db.json",
+   autoload: true,
+});
+
+export default db;
diff --git a/registry/src/deps.ts b/registry/src/deps.ts
new file mode 100644
index 0000000..f630d34
--- /dev/null
+++ b/registry/src/deps.ts
@@ -0,0 +1,18 @@
+// export { MongoClient, ObjectId } from "https://deno.land/x/mongo@v0.9.1/mod.ts";
+
+export * as S3 from "https://deno.land/x/s3/mod.ts";
+
+export * as Ini from "https://deno.land/x/ini/mod.ts";
+
+export * as ABC from "https://deno.land/x/abc@v1/mod.ts";
+export * as CorsMW from "https://deno.land/x/abc@v1/middleware/cors.ts";
+export * as LoggerMW from "https://deno.land/x/abc@v1/middleware/logger.ts";
+
+export * as Path from "https://deno.land/std@0.62.0/path/mod.ts";
+export * as FS from "https://deno.land/std@0.62.0/fs/mod.ts";
+
+export * as Compress from "https://git.stamm.me/Deno/DenReg/raw/branch/master/tar/mod.ts";
+
+import DS from "https://raw.githubusercontent.com/hibas123/dndb/master/mod.ts";
+
+export const Datastore = DS;
diff --git a/registry/src/http.ts b/registry/src/http.ts
new file mode 100644
index 0000000..e6b9023
--- /dev/null
+++ b/registry/src/http.ts
@@ -0,0 +1,18 @@
+import { ABC, CorsMW, LoggerMW } from "./deps.ts";
+import config from "./config.ts";
+
+const port = config?.api?.port || 8000;
+const app = new ABC.Application();
+
+app.use(LoggerMW.logger({}));
+app.use(CorsMW.cors({}));
+
+import api from "./http/api.ts";
+api(app.group("api"));
+
+import raw from "./http/raw.ts";
+raw(app.group("raw"));
+
+app.start({ port });
+console.log("Running server at http://0.0.0.0:" + port);
+console.log("Open at http://127.0.0.1:" + port);
diff --git a/registry/src/http/api.ts b/registry/src/http/api.ts
new file mode 100644
index 0000000..ded483a
--- /dev/null
+++ b/registry/src/http/api.ts
@@ -0,0 +1,133 @@
+import { ABC, Path, Compress, FS } from "../deps.ts";
+
+import bucket from "../s3.ts";
+import { isValidPackageName, basicauth, isValidFullVersion } from "../utils.ts";
+
+import db, { IPackage } from "../db.ts";
+
+import { v4 } from "https://deno.land/std/uuid/mod.ts";
+
+export default function api(g: ABC.Group) {
+   g.get("/", (ctx) => {
+      return { version: "1" };
+   });
+
+   g.post("/package/:name", uploadPackage, basicauth("api"));
+}
+
+async function uploadPackage(ctx: ABC.Context) {
+   const reqId = v4.generate();
+   const filename = "./tmp/" + reqId + ".tar";
+   const folder = "./tmp/" + reqId;
+
+   try {
+      const packageName = ctx.params.name;
+
+      if (!isValidPackageName(packageName))
+         throw new Error("Invalid package name");
+
+      console.log("Writing body to tmp file:", filename);
+      const file = await Deno.open(filename, {
+         write: true,
+         create: true,
+      });
+      await Deno.copy(ctx.request.body, file);
+      file.close();
+
+      console.log("Create unpacked folder");
+
+      await Deno.mkdir(folder);
+
+      console.log("Uncompressing tar");
+
+      await Compress.Tar.uncompress(filename, folder);
+
+      console.log("Reading meta.json");
+
+      const fileContent = await Deno.readFile(Path.join(folder, "meta.json"));
+
+      console.log("Parsing meta.json");
+
+      const meta = JSON.parse(new TextDecoder().decode(fileContent));
+
+      console.log("Checking meta.json");
+
+      if (!meta?.version) {
+         throw new Error("No version available in meta.json");
+      }
+
+      const packageVersion = meta.version;
+
+      console.log("Checking correct version");
+
+      if (!isValidFullVersion(packageVersion)) {
+         throw new Error("Invalid version. Version must be in format: 0.0.0");
+      }
+
+      console.log("Checking for previous uploads");
+
+      let packageMeta = await db.findOne({ name: packageName });
+
+      if (!packageMeta) {
+         packageMeta = {
+            name: packageName,
+            versions: [],
+         };
+
+         await db.insert(packageMeta);
+      }
+
+      console.log("Check if version was uploaded before");
+
+      if (packageMeta.versions.find((e) => e === meta.version)) {
+         throw new Error("Version was already uploaded!");
+      }
+
+      const bucketBase = "packages/" + packageName + "/" + packageVersion + "/";
+
+      console.log("Uploading files to S3");
+
+      const walker = FS.walk(folder, {
+         includeFiles: true,
+         includeDirs: false,
+      });
+
+      for await (const file of walker) {
+         const relative = Path.relative(folder, file.path);
+
+         const bucketPath = (bucketBase + relative).replace(/@/g, "§");
+
+         console.log("Uploading file:", file.path, bucketPath, bucketBase);
+         await bucket.putObject(
+            bucketPath,
+            await Deno.readAll(await Deno.open(file.path)),
+            {}
+         );
+      }
+      console.log("Setting new live version");
+
+      //TODO: Better option, since this could error whith multiple upload to the same package
+      await db.update(
+         { name: packageName },
+         {
+            $set: { versions: [...packageMeta.versions, packageVersion] },
+         }
+      );
+
+      console.log("Finished successfully");
+   } catch (err) {
+      console.error("Error while processing newly uploaded package");
+      console.error(err);
+      return {
+         success: false,
+         message: err.message,
+      };
+   } finally {
+      console.log("Cleanup");
+      await Deno.remove(filename).catch(console.error);
+      await Deno.remove(folder, { recursive: true }).catch(console.error);
+   }
+   return {
+      success: true,
+   };
+}
diff --git a/registry/src/http/raw.ts b/registry/src/http/raw.ts
new file mode 100644
index 0000000..0ef2226
--- /dev/null
+++ b/registry/src/http/raw.ts
@@ -0,0 +1,53 @@
+import { ABC } from "../deps.ts";
+
+import { sortVersions, extractPackagePath } from "../utils.ts";
+
+import db, { IPackage } from "../db.ts";
+
+import bucket from "../s3.ts";
+
+export default function raw(g: ABC.Group) {
+   g.get("/:package/*path", async (ctx) => {
+      console.log(ctx.params, ctx.path);
+      let [packageName, packageVersion] = extractPackagePath(
+         ctx.params.package
+      );
+
+      const meta = await db.findOne({ name: packageName });
+
+      console.log(packageName, await db.findOne({ name: packageName }));
+
+      const E404 = () => {
+         ctx.response.status = 404;
+         ctx.response.body = "Not found!";
+         throw new Error("Not found!");
+      };
+
+      if (!meta || meta.versions.length < 1) return E404();
+
+      const versions = meta.versions.sort(sortVersions).reverse();
+
+      if (!packageVersion) {
+         packageVersion = versions[0];
+      } else {
+         const v = versions.filter((e) =>
+            e.startsWith(packageVersion as string)
+         );
+         if (v.length < 1) return E404();
+         packageVersion = v[0];
+      }
+
+      const bucketPath = (
+         "packages/" +
+         packageName +
+         "/" +
+         packageVersion +
+         "/" +
+         ctx.params.path
+      ).replace(/@/g, "§");
+
+      console.log("Getting file from:", bucketPath);
+
+      return (await bucket.getObject(bucketPath))?.body;
+   });
+}
diff --git a/registry/src/registry.ts b/registry/src/registry.ts
new file mode 100644
index 0000000..7b5347d
--- /dev/null
+++ b/registry/src/registry.ts
@@ -0,0 +1,18 @@
+const ensureDir = async (name: string) => {
+   if (
+      !(await Deno.stat(name)
+         .then(() => true)
+         .catch(() => false))
+   ) {
+      await Deno.mkdir(name);
+   }
+};
+
+try {
+   await Deno.remove("./tmp", { recursive: true });
+} catch (err) {}
+
+await ensureDir("./tmp");
+await ensureDir("./data");
+
+import "./http.ts";
diff --git a/registry/src/s3.ts b/registry/src/s3.ts
new file mode 100644
index 0000000..c014bdc
--- /dev/null
+++ b/registry/src/s3.ts
@@ -0,0 +1,16 @@
+import { S3 } from "./deps.ts";
+import config from "./config.ts";
+
+if (!config.s3) {
+   throw new Error("Config is missing [s3] section!");
+}
+
+const bucket = new S3.S3Bucket({
+   bucket: config.s3.bucket || "deno-registry",
+   endpointURL: config.s3.endpoint,
+   accessKeyID: config.s3.accessKey,
+   secretKey: config.s3.secretKey,
+   region: config?.s3?.region || "us-east-1",
+});
+
+export default bucket;
diff --git a/registry/src/utils.ts b/registry/src/utils.ts
new file mode 100644
index 0000000..20f49c6
--- /dev/null
+++ b/registry/src/utils.ts
@@ -0,0 +1,86 @@
+import { ABC } from "./deps.ts";
+import { decode } from "https://deno.land/std/encoding/base64.ts";
+import config from "./config.ts";
+
+const packageNameRegex = /^[@]?[a-zA-Z][\d\w\-\_]*$/g.compile();
+const packageVersionRegex = /^\d(\.\d)?(\.\d)?$/g.compile();
+const packageFullVersionRegex = /^\d(\.\d)(\.\d)$/g.compile();
+
+export const isValidPackageName = (name: string) => packageNameRegex.test(name);
+export const isValidVersion = (version: string) =>
+   packageVersionRegex.test(version);
+export const isValidFullVersion = (version: string) =>
+   packageFullVersionRegex.test(version);
+
+const ALg = 1;
+const ASm = -ALg;
+const Equ = 0;
+
+export const sortVersions = (a: string, b: string) => {
+   const [a1, a2, a3] = a.split(".").map(Number);
+   const [b1, b2, b3] = b.split(".").map(Number);
+
+   if (a1 > b1) return ALg;
+   if (a1 < b1) return ASm;
+
+   if (a2 > b2) return ALg;
+   if (a2 < b2) return ASm;
+
+   if (a3 > b3) return ALg;
+   if (a3 < b3) return ASm;
+
+   return Equ;
+};
+
+export const basicauth = (realm: string) => (next: ABC.HandlerFunc) => (
+   ctx: ABC.Context
+) => {
+   const value = ctx.request.headers.get("authorization");
+
+   console.log("Header:", value);
+
+   if (value && value.toLowerCase().startsWith("basic ")) {
+      const credentials = value.slice(6);
+      const [username, passwd] = new TextDecoder()
+         .decode(decode(credentials))
+         .split(":", 2);
+
+      if (config?.user[username]?.password === passwd) {
+         console.log("User authenticated!");
+         return next(ctx);
+      }
+   }
+
+   console.log("Authentication required");
+   ctx.response.status = 401;
+   ctx.response.headers.set("WWW-Authenticate", "Basic realm=" + realm);
+   return {
+      statusCode: 401,
+      error: "Authentication required",
+   };
+};
+
+export function extractPackagePath(path: string): [string, string | undefined] {
+   let packageName = "";
+   if (path.startsWith("@")) {
+      packageName = "@";
+      path = path.slice(1);
+   }
+
+   let parts = path.split("@");
+   if (parts.length > 2) throw new Error("Invalid package name!");
+
+   packageName += parts[0];
+   let packageVersion: string | undefined = path[1];
+
+   if (!isValidPackageName(packageName))
+      throw new Error("Invalid package name!");
+
+   if (packageVersion !== "") {
+      if (!isValidVersion(packageVersion))
+         throw new Error("Invalid package version!");
+      else packageVersion = undefined;
+   }
+
+   return [packageName, packageVersion];
+}