Deno/DenReg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

Make it modern

Browse Source
This commit is contained in:
Fabian Stamm
2023-11-28 16:10:33 +01:00
parent 22a447604b
commit a14a5b9462
35 changed files with 656 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
registry/src/deps.ts
View File

@ -1,8 +1,4 @@
// @deno-types="./types/hotfix.d.ts"
export * as S3 from "https://deno.land/x/s3@0.3.0/mod.ts";
export { S3Error } from "https://deno.land/x/s3@0.3.0/src/error.ts";
export * as Ini from "https://deno.hibas123.de/raw/ini@0.0.3/mod.ts";
export * as ABC from "https://deno.land/x/abc@v1.2.4/mod.ts";
@ -11,20 +7,20 @@ export * as LoggerMW from "https://deno.land/x/abc@v1.2.4/middleware/logger.ts";
export * as Path from "https://deno.land/std@0.83.0/path/mod.ts";
export * as FS from "https://deno.land/std@0.83.0/fs/mod.ts";
export * as Base64 from "https://deno.land/std@0.83.0/encoding/base64.ts";
export * as Base64 from "https://deno.land/std@0.208.0/encoding/base64.ts";
export * as Hash from "https://deno.land/std@0.83.0/hash/mod.ts";
export * as Colors from "https://deno.land/std@0.83.0/fmt/colors.ts";
export * as Streams from "https://deno.land/std@0.208.0/streams/mod.ts";
export * as Compress from "https://git.stamm.me/Deno/DenReg/raw/branch/master/tar/mod.ts";
export * as Compress from "https://git.hibas.dev/Deno/DenReg/raw/branch/master/tar/mod.ts";
export { default as Prism } from "https://cdn.skypack.dev/prismjs";
export { Marked } from "https://deno.hibas123.de/raw/markdown/mod.ts";
export { Marked } from "https://deno.hibas123.de/raw/markdown@0.1.0/mod.ts";
import DS from "https://raw.githubusercontent.com/hibas123/dndb/master/mod.ts";
import * as Pico from "https://deno.hibas123.de/raw/@denreg-jsx@0.1.2/mod.ts";
export { Pico };
export * as Nano from "https://deno.land/x/nano_jsx@v0.1.0/mod.ts";
export const Datastore = DS;

44
registry/src/http/api.ts
View File

@ -1,17 +1,16 @@
import { ABC, Path, Compress, FS, Colors, S3Error } from "../deps.ts";
import { ABC, Path, Compress, FS, Colors } from "../deps.ts";
import bucket from "../s3.ts";
import {
isValidPackageName,
basicauth,
isValidFullVersion,
getAbsolutePackageVersion,
getBucketFilePath,
getFilePath,
} from "../utils.ts";
import db, { IPackage } from "../db.ts";
import * as Storage from "../storage.ts";
import { v4 } from "https://deno.land/std/uuid/mod.ts";
import db from "../db.ts";
export default function api(g: ABC.Group) {
const cacheControl = (next: ABC.HandlerFunc) => (ctx: ABC.Context) => {
@ -21,14 +20,15 @@ export default function api(g: ABC.Group) {
g.get(
"/",
(ctx) => {
(_ctx) => {
return { version: "1" };
},
cacheControl
);
g.get("/module", async (ctx) => {
return db.package.find({}).then((res) => res.map((e) => e.name));
g.get("/module", async (_ctx) => {
const res = await db.package.find({});
return res.map((e) => e.name);
});
g.get("/module/:module", async (ctx) => {
@ -47,7 +47,7 @@ export default function api(g: ABC.Group) {
ctx.response.status = 404;
return "// Not found";
} else {
let version = getAbsolutePackageVersion(
const version = getAbsolutePackageVersion(
module,
ctx.params.version
) as string;
@ -57,19 +57,17 @@ export default function api(g: ABC.Group) {
return "// Not found";
}
const bucketPath = await getBucketFilePath(module.name, version, "/");
const bucketPath = await getFilePath(module.name, version, "/");
const filesItr = Storage.walkFiles(module.name + "/" + version + "/");
const filesItr = bucket.listAllObjects({
batchSize: 100,
prefix: bucketPath,
});
const allowedExts = new Set(
(ctx.queryParams.ext || "js|ts").split("|").map((e) => "." + e)
);
let files: string[] = [];
for await (let file of filesItr) {
const relPath = Path.posix.relative(bucketPath, file.key || "");
const files: string[] = [];
for await (const file of filesItr) {
const relPath = Path.posix.relative(bucketPath, file.path || "");
const ext = Path.extname(relPath);
if (allowedExts.has(ext)) files.push(relPath);
}
@ -100,7 +98,7 @@ export default function api(g: ABC.Group) {
// }
async function uploadPackage(ctx: ABC.Context) {
const reqId = v4.generate();
const reqId = crypto.randomUUID();
const filename = "./tmp/" + reqId + ".tar";
const folder = "./tmp/" + reqId;
@ -119,6 +117,7 @@ async function uploadPackage(ctx: ABC.Context) {
write: true,
create: true,
});
// ctx.request.body.pipeTo(file); //TODO: Do this, once web framework is updated
await Deno.copy(ctx.request.body, file);
file.close();
@ -156,7 +155,6 @@ async function uploadPackage(ctx: ABC.Context) {
success: false,
message: "Invalid version. Version must be in format: 0.0.0",
};
return;
}
console.log("Checking for previous uploads");
@ -187,7 +185,7 @@ async function uploadPackage(ctx: ABC.Context) {
};
}
const bucketBase = "packages/" + packageName + "/" + packageVersion + "/";
const storageBase = packageName + "/" + packageVersion + "/";
console.log("Uploading files to S3");
@ -201,11 +199,12 @@ async function uploadPackage(ctx: ABC.Context) {
console.log("Normalised path:", relative.replace("\\", "/"));
const bucketPath = (bucketBase + relative).replace(/@/g, "§");
const bucketPath = (storageBase + relative).replace(/@/g, "§");
const body = await Deno.readAll(await Deno.open(file.path));
console.log("Put Object", bucketPath, body.byteLength);
await bucket.putObject(bucketPath, body, {});
await Storage.writeFile(bucketPath, body);
}
console.log("Setting new live version");
@ -230,7 +229,6 @@ async function uploadPackage(ctx: ABC.Context) {
Colors.red("Error while processing newly uploaded package")
);
console.error(err);
if (err instanceof S3Error) console.log(err.response);
return {
success: false,
message: err.message,

2
registry/src/http/intellisense.ts
View File

@ -3,7 +3,7 @@ import { ABC } from "../deps.ts";
import config from "../config.ts";
export default function raw(g: ABC.Group) {
g.get("/deno-import-intellisense.json", (ctx) => {
g.get("/deno-import-intellisense.json", (_ctx) => {
return {
version: 1,
registries: [

2
registry/src/http/raw.ts
View File

@ -30,7 +30,7 @@ export default function raw(g: ABC.Group) {
const result = await getFile(packageName, packageVersion, filepath);
if (filepath.endsWith(".js")) {
const tsFile = filepath.substr(0, filepath.length - 3) + ".d.ts";
const tsFile = filepath.slice(0, filepath.length - 3) + ".d.ts";
const tsResult = await getFile(packageName, packageVersion, tsFile);
if (tsResult) {
ctx.response.headers.set(

53
registry/src/http/views.ts
View File

@ -1,16 +1,16 @@
import type { ABC } from "../deps.ts";
import {
basicauth,
extractPackagePath,
getBucketFilePath,
getFilePath,
getFile,
getOneOf,
getAbsolutePackageVersion,
sortVersions,
} from "../utils.ts";
import { Hash, Path } from "../deps.ts";
import db, { IPackage } from "../db.ts";
import bucket from "../s3.ts";
import * as Storage from "../storage.ts";
const MAX_CACHE_AGE = 60 * 30; // 30 Minutes
@ -31,6 +31,7 @@ export default function views(g: ABC.Application) {
packages = await db.package.find({});
}
await ctx.render("index", {
packages: packages.reverse(),
search,
@ -54,19 +55,36 @@ export default function views(g: ABC.Application) {
});
g.get("/package/:package", async (ctx) => {
let [packageName, packageVersion] = extractPackagePath(
const [packageName, packageVersion] = extractPackagePath(
ctx.params.package
);
const pkg = await db.package.findOne({ name: packageName });
if (!pkg) {
ctx.response.status = 404;
ctx.response.body = "// Package not found!";
return;
}
const readmeContentRaw = (await getOneOf(
pkg.name,
packageVersion || pkg.versions.sort(sortVersions).reverse()[0],
[pkg.readme, "README.md", "readme.md", "Readme.md"]
))?.data;
const readmeContent = readmeContentRaw
? new TextDecoder().decode(readmeContentRaw)
: "";
const etag =
"W/" +
Hash.createHash("sha3-256")
.update(`${packageName}:${packageVersion}`)
.toString("base64");
await ctx.render("package", { pkg, version: packageVersion });
await ctx.render("package", { pkg, version: packageVersion, readmeContent });
ctx.response.headers.set("cache-control", CACHE_CONTROL);
ctx.response.headers.set("E-Tag", etag);
});
@ -98,32 +116,29 @@ export default function views(g: ABC.Application) {
path: `${packageName}@${packageVersion}/${path}`,
});
} else {
const bucketPath = await getBucketFilePath(
const filesPath = getFilePath(
packageName,
packageVersion,
path
);
if (!bucketPath) return E404();
if (!filesPath) return E404();
console.log(bucketPath);
console.log(filesPath);
const filesItr = bucket.listAllObjects({
batchSize: 100,
prefix: bucketPath,
// delimiter: "/",
});
const filesItr = Storage.walkFiles(filesPath);
let files: { name: string; size: number }[] = [];
let directories: Set<string> = new Set();
const files: { name: string; size: number }[] = [];
const directories: Set<string> = new Set();
let readme: string | null = null;
for await (let file of filesItr) {
const relPath = Path.posix.relative(bucketPath, file.key || "");
for await (const file of filesItr) {
const relPath = Path.posix.relative(filesPath, file.path || "");
console.log({ file, relPath, filesPath });
if (relPath.indexOf("/") >= 0) {
directories.add(relPath.split("/")[0]);
} else {
files.push({ name: relPath, size: file.size || -1 });
files.push({ name: relPath, size: -1 }); //TODO: Size is not implemented yet
if (relPath.toLowerCase() === "readme.md") {
let readmeCont = await getFile(
const readmeCont = await getFile(
packageName,
packageVersion,
Path.posix.join(path, relPath)

0
registry/src/react.ts Normal file
View File

13
registry/src/renderer.tsx
View File

@ -1,11 +1,9 @@
// / <reference path="./types/jsx.d.ts" />
import { Pico } from "./deps.ts";
import { Nano } from "./deps.ts";
const { h, renderSSR } = Nano;
import config from "./config.ts";
const React = {
createElement: Pico.h.bind(Pico),
};
class StringReader implements Deno.Reader {
private data: Uint8Array;
@ -70,8 +68,9 @@ export default async function render(
): Promise<Deno.Reader> {
const Component = await loadComponent(name);
//@ts-ignore
const res = await Pico.renderSSR(<Component {...data} />);
// // @ts-ignore
// const res = await Pico.renderSSR(<Component {...data} />);
const res = await renderSSR(<Component {...data} />);
return new StringReader("<!DOCTYPE html>\n" + res);
}

30
registry/src/s3.ts
View File

@ -1,30 +0,0 @@
import { S3 } from "./deps.ts";
import config from "./config.ts";
if (!config.s3) {
throw new Error("Config is missing [s3] section!");
}
if (!config.s3.endpoint) {
throw new Error("Config is missing s3.endpoint!");
}
if (!config.s3.accessKey) {
throw new Error("Config is missing s3.accessKey!");
}
if (!config.s3.secretKey) {
throw new Error("Config is missing s3.secretKey!");
}
const s3config: S3.S3BucketConfig = {
bucket: config.s3.bucket || "deno-registry",
endpointURL: config.s3.endpoint,
accessKeyID: config.s3.accessKey,
secretKey: config.s3.secretKey,
region: config?.s3?.region || "us-east-1",
};
const bucket = new S3.S3Bucket(s3config);
export default bucket;

40
registry/src/storage.ts Normal file
View File

@ -0,0 +1,40 @@
import { FS, Path } from "./deps.ts";
const STORAGE_ROOT = Path.resolve("./data/files");
export function walkFiles(path: string) {
const norm = Path.normalize(path);
const p = Path.join(STORAGE_ROOT, norm);
const walker = FS.walk(p, { includeFiles: true, includeDirs: false })
// Make path of entry relative to the root and return an async iterator
return (async function* () {
for await (const entry of walker) {
const rel = Path.relative(STORAGE_ROOT, entry.path);
yield { ...entry, path: rel };
}
})();
}
export async function writeFile(path: string, content: Uint8Array) {
const norm = Path.normalize(path);
const p = Path.join(STORAGE_ROOT, norm);
await FS.ensureFile(p);
await Deno.writeFile(p, content);
}
export async function readFile(path: string) {
const norm = Path.normalize(path);
const p = Path.join(STORAGE_ROOT, norm);
if (!await FS.exists(p)) return undefined;
const s = await Deno.stat(p);
if (!s.isFile) return undefined;
return await Deno.readFile(p);
}

42
registry/src/utils.ts
View File

@ -1,15 +1,13 @@
import { ABC, Base64, Path } from "./deps.ts";
import config from "./config.ts";
import * as Storage from "./storage.ts";
const packageNameRegex = /^[@]?[a-zA-Z][\d\w\-\_]*$/g.compile();
const packageVersionRegex = /^\d(\.\d)?(\.\d)?$/g.compile();
const packageFullVersionRegex = /^\d(\.\d)(\.\d)$/g.compile();
export const isValidPackageName = (name: string) => packageNameRegex.test(name);
export const isValidPackageName = (name: string) => /^[@]?[a-zA-Z][\d\w\-\_]*$/g.test(name);
export const isValidVersion = (version: string) =>
packageVersionRegex.test(version);
/^\d+(\.\d+)?(\.\d+)?$/g.test(version);
export const isValidFullVersion = (version: string) =>
packageFullVersionRegex.test(version);
/^\d+(\.\d+)(\.\d+)$/g.test(version);
const ALg = 1;
const ASm = -ALg;
@ -70,16 +68,18 @@ export function extractPackagePath(path: string): [string, string | undefined] {
path = path.slice(1);
}
let parts = path.split("@");
const parts = path.split("@");
if (parts.length > 2) throw new Error("Invalid package name!");
packageName += parts[0];
let packageVersion: string | undefined = parts[1];
const packageVersion: string | undefined = parts[1];
console.log({ path, parts, packageName, packageVersion });
if (!isValidPackageName(packageName))
throw new Error("Invalid package name!");
if (packageVersion !== "") {
if (packageVersion && packageVersion !== "") {
if (!isValidVersion(packageVersion))
throw new Error("Invalid package version!");
}
@ -89,8 +89,6 @@ export function extractPackagePath(path: string): [string, string | undefined] {
import type { IPackage } from "./db.ts";
import bucket from "./s3.ts";
export function getAbsolutePackageVersion(
pkg?: IPackage | null,
version?: string
@ -110,7 +108,7 @@ export function getAbsolutePackageVersion(
return version;
}
export async function getBucketFilePath(
export function getFilePath(
pkgName: string,
version: string,
file: string
@ -120,7 +118,6 @@ export async function getBucketFilePath(
}
const bucketPath = (
"packages/" +
pkgName +
"/" +
version +
@ -131,24 +128,33 @@ export async function getBucketFilePath(
return bucketPath;
}
export async function getOneOf(pkgName: string, version: string, files: (string | undefined)[]) {
for (const file of files) {
if (!file) continue;
const res = await getFile(pkgName, version, file);
if (res) return res;
}
return undefined;
}
export async function getFile(
pkgName: string,
version: string | undefined,
file: string
): Promise<{ etag: string; data: Uint8Array } | null | undefined> {
if (!version) return undefined;
const bucketPath = await getBucketFilePath(pkgName, version, file);
const bucketPath = await getFilePath(pkgName, version, file);
if (!bucketPath) return null;
console.log("Getting file from:", bucketPath);
try {
const res = await bucket.getObject(bucketPath);
if (!res || res.body.byteLength === 0) return undefined;
const res = await Storage.readFile(bucketPath);
if (!res) return undefined;
return {
etag: res.etag,
data: res.body,
etag: Base64.encodeBase64(await crypto.subtle.digest("sha-1", res)),
data: res,
};
} catch (err) {
const msg = err.message as string;

13
registry/src/views/_base.tsx
View File

@ -1,4 +1,6 @@
import { Pico } from "../deps.ts";
import { Nano } from "../deps.ts";
const { h } = Nano;
import config from "../config.ts";
const styles = new TextDecoder().decode(
@ -6,12 +8,13 @@ const styles = new TextDecoder().decode(
);
// href="https://unpkg.com/papercss@1.6.1/dist/paper.min.css"
export default function Base(p: any, children: any[]) {
export default function Base(p: any) {
const title = p.title || "DenReg";
return (
<html lang="en">
<head>
<meta charset="UTF-8"></meta>
<style dangerouslySetInnerHTML={{ __html: ".clean {all:revert;}" }} />
{/* <link
rel="stylesheet"
href="https://deno.hibas123.de/raw/@hibas123-theme@2.0.2/out/base.css"
@ -94,7 +97,7 @@ export default function Base(p: any, children: any[]) {
/>
<meta name="theme-color" content="#ffffff"></meta>
<link href="/public/prism.css" rel="stylesheet" />
<style innerHTML={styles}></style>
<style dangerouslySetInnerHTML={{ __html: styles }}></style>
<title>{title}</title>
<meta
name="Description"
@ -107,9 +110,9 @@ export default function Base(p: any, children: any[]) {
</head>
<body class="site">
{config.web.tracking && (
<tracking innerHTML={config.web.tracking}></tracking>
<tracking dangerouslySetInnerHTML={{ __html: config.web.tracking }}></tracking>
)}
{children}
{p.children}
</body>
</html>
);

8
registry/src/views/_browse.tsx
View File

@ -1,4 +1,6 @@
import { Pico, Marked } from "../deps.ts";
import { Marked } from "../deps.ts";
import { Nano } from "../deps.ts";
const { h, Fragment } = Nano;
import type { IPackage } from "../db.ts";
import { sortVersions } from "../utils.ts";
@ -67,7 +69,7 @@ export function RenderFile({ content, ext }: IRenderFileInterface) {
<div
class="card browse-code-block"
style="margin-top: 1rem; padding: 1rem;"
innerHTML={content}
dangerouslySetInnerHTML={{ __html: content }}
/>
);
} else {
@ -77,7 +79,7 @@ export function RenderFile({ content, ext }: IRenderFileInterface) {
content = Prism.highlight(content, Prism.languages[lang], lang);
}
return <pre innerHTML={content} />;
return <pre><code dangerouslySetInnerHTML={{ __html: content }}> </code></pre>;
}
}

11
registry/src/views/_default.tsx
View File

@ -1,14 +1,15 @@
import { Pico } from "../deps.ts";
import { Nano } from "../deps.ts";
const { h } = Nano;
export function Main(a: any, children: any) {
export function Main(a: any) {
return (
<div style="grid-area: main">
<div class="paper">{children}</div>
<div class="paper">{a.children}</div>
</div>
);
}
export function Menu({}: any, children: any) {
export function Menu(a: any) {
return (
<div style="grid-area: menu">
<div class="paper">
@ -22,7 +23,7 @@ export function Menu({}: any, children: any) {
</a>
</h3>
{children}
{a.children}
</div>
</div>
);

6
registry/src/views/_pkgheader.tsx
View File

@ -1,8 +1,10 @@
// /// <reference path="../types/jsx.d.ts" />
import { Pico, Marked } from "../deps.ts";
import { Nano } from "../deps.ts";
const { h, Fragment } = Nano;
import type { IPackage } from "../db.ts";
export default async function index({
export default function index({
pkg,
version,
}: {

5
registry/src/views/browse_file.tsx
View File

@ -1,11 +1,12 @@
import { Pico } from "../deps.ts";
import { Nano } from "../deps.ts";
const { h } = Nano;
import Base from "./_base.tsx";
import type { IPackage } from "../db.ts";
import { Main, Menu } from "./_default.tsx";
import { RenderFile, EntryList, BrowseHeader } from "./_browse.tsx";
export default async function index({
export default function index({
pkg,
version,
content,

5
registry/src/views/browse_folder.tsx
View File

@ -1,11 +1,12 @@
import { Pico } from "../deps.ts";
import { Nano } from "../deps.ts";
const { h } = Nano;
import Base from "./_base.tsx";
import type { IPackage } from "../db.ts";
import { Main, Menu } from "./_default.tsx";
import { RenderFile, EntryList, BrowseHeader } from "./_browse.tsx";
export default async function index({
export default function index({
pkg,
version,
files,

15
registry/src/views/index.tsx
View File

@ -1,4 +1,5 @@
import { Pico } from "../deps.ts";
import { Nano } from "../deps.ts";
const { h } = Nano;
import Base from "./_base.tsx";
import type { IPackage } from "../db.ts";
import { sortVersions } from "../utils.ts";
@ -9,9 +10,11 @@ function Package({ pkg }: { pkg: IPackage }) {
const sorted = versions.sort(sortVersions).reverse();
return (
<div
class="card package-list-margin"
onClick={"window.location.href = '/package/" + name + "'"}
<a
style="text-decoration:none; color: black"
class="clean card package-list-margin"
href={"/package/" + name}
// onClick={"window.location.href = '/package/" + name + "'"}
>
<div class="card-body">
<h4 class="card-title">
@ -26,13 +29,13 @@ function Package({ pkg }: { pkg: IPackage }) {
{description}
</div>
</div>
</div>
</a>
);
}
import { Main, Menu } from "./_default.tsx";
export default async function index({
export default function index({
packages,
search,
}: {

21
registry/src/views/package.tsx
View File

@ -1,4 +1,5 @@
import { Pico, Marked } from "../deps.ts";
import { Nano, Marked } from "../deps.ts";
const { h, Suspense } = Nano;
import Base from "./_base.tsx";
import type { IPackage } from "../db.ts";
import { sortVersions, getFile, getAbsolutePackageVersion } from "../utils.ts";
@ -29,12 +30,14 @@ import PkgHeader from "./_pkgheader.tsx";
import { Main, Menu } from "./_default.tsx";
export default async function index({
export default function index({
pkg,
version,
readmeContent
}: {
pkg: IPackage;
version?: string;
readmeContent: string | undefined;
}) {
if (!pkg)
return (
@ -44,16 +47,6 @@ export default async function index({
);
version = getAbsolutePackageVersion(pkg, version);
const readmeContent = await getFile(
pkg.name,
version,
pkg.readme || "README.md"
).then((res) => {
if (res)
return Marked.parse(new TextDecoder().decode(res.data))
.content as string;
else return undefined;
});
return (
<Base title={"DenReg - " + pkg.name}>
@ -69,7 +62,7 @@ export default async function index({
{readmeContent !== undefined ? (
<div
style="overflow-x: hidden"
innerHTML={readmeContent}
dangerouslySetInnerHTML={{ __html: Marked.parse(readmeContent).content }}
/>
) : (
<div class="alert alert-warning">No README.md found!</div>
@ -94,4 +87,4 @@ export default async function index({
</Menu>
</Base>
);
}
}