hibas123/SimpleVSC
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Adding tests and fixing stuff

Browse Source
This commit is contained in:
user
2021-08-10 18:03:20 +02:00
parent cf1347ca3a
commit f3f3d250b4
6 changed files with 754 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/datasources/fs.ts
View File

@ -10,6 +10,8 @@ export default class FSDataStore implements IDataStore {
}
async get(name: string) {
if (!await this.has(name))
return undefined;
return Fs.promises.readFile(Path.join(this.#path, name));
}
@ -34,11 +36,11 @@ export default class FSDataStore implements IDataStore {
//TODO: This is not atomic yet! FIX
while ((await this.has("lock")) || this.#l) {
try {
let con = (await this.get("lock")).toString("utf-8");
let con = (await this.get("lock") as Buffer).toString("utf-8");
if (Number(con) < Date.now() - 10000) break;
await new Promise((y) => setTimeout(y, 10));
} catch (err) {}
} catch (err) { }
}
this.#l = true;

291
src/repo.ts
View File

@ -2,7 +2,7 @@ import SHA from "jssha";
import Path from "./helper/path";
export interface IDataStore {
get(key: string): Promise<Uint8Array>;
get(key: string): Promise<Uint8Array | undefined>;
set(key: string, data: Uint8Array): Promise<void>;
has(key: string): Promise<boolean>;
close?: () => Promise<void>;
@ -22,6 +22,14 @@ export type Commit = {
date: Date;
};
export type NodeLog = {
/**
* ObjectID of the data. Can be undefined if element was deleted!
*/
id: string | undefined;
commit: Commit
}
// TODOs:
// - HEAD locks
// - HEAD/Tree Cache
@ -29,12 +37,16 @@ export type Commit = {
// - Add DataStore Locking for access from multiple sources
export default class Repository {
//#region local variables
#store: IDataStore;
//#endregion
constructor(store: IDataStore) {
this.#store = store;
}
//#region private
private sha1(data: Uint8Array) {
const s = new SHA("SHA-1", "UINT8ARRAY");
s.update(data);
@ -50,11 +62,9 @@ export default class Repository {
private splitPath(path: string) {
const resolved = Path.resolve(path).slice(1);
if (resolved == "") return [];
return resolved.split(Path.delimiter);
return resolved.split(Path.sep);
}
private getHeadLock() {}
private async writeObject(data: Uint8Array | string): Promise<string> {
if (typeof data == "string") {
data = new TextEncoder().encode(data);
@ -68,9 +78,9 @@ export default class Repository {
return this.#store.has("objects/" + id);
}
private async readObject(id: string, string: true): Promise<string>;
private async readObject(id: string, string?: false): Promise<Uint8Array>;
private async readObject(id: string, string = false): Promise<Uint8Array | string> {
private async readObject(id: string, string: true): Promise<string | undefined>;
private async readObject(id: string, string?: false): Promise<Uint8Array | undefined>;
private async readObject(id: string, string = false): Promise<Uint8Array | string | undefined> {
let data = await this.#store.get("objects/" + id);
if (string) {
return new TextDecoder().decode(data);
@ -79,94 +89,7 @@ export default class Repository {
}
}
async clean() {
// TODO: Cleanup broken things
}
async readdir(path: string): Promise<TreeEntry[] | undefined> {
const parts = this.splitPath(path);
console.log({ parts });
const head = await this.readHead();
if (!head) return undefined;
const treeID = await this.treeFindObjectID(head.root, parts, "tree");
if (!treeID) return undefined;
return this.readTree(treeID);
}
async read(path: string): Promise<Uint8Array | undefined> {
const parts = this.splitPath(path);
const head = await this.readHead();
if (!head) return undefined;
const objectID = await this.treeFindObjectID(head.root, parts, "blob");
if (!objectID) return undefined;
return this.readObject(objectID);
}
async write(path: string, data: Uint8Array) {
const parts = this.splitPath(path);
const objectID = await this.writeObject(data);
const lock = await this.#store.getLock();
try {
//TODO: Improve need of locking.
const head = await this.readHead();
const makeTree = async (treeID: string | undefined, parts: string[]) => {
let tree: TreeEntry[];
if (treeID) {
tree = await this.readTree(treeID);
} else {
tree = [];
}
const current = parts[0];
let existing = tree.findIndex(([, , name]) => name == current);
let entry: TreeEntry;
if (parts.length == 1) {
entry = ["blob", objectID, current];
} else {
entry = [
"tree",
await makeTree(existing >= 0 ? tree[existing][1] : undefined, parts.slice(1)),
current,
];
}
if (existing >= 0) {
let ex = tree[existing];
if (parts.length == 1 && ex[0] == "tree")
throw new Error("This change would overwrite a folder!");
tree[existing] = entry;
} else {
tree.push(entry);
}
let treeString = tree.map(([type, hash, name]) => `${type} ${hash} ${name}`).join("\n");
let newTreeID = await this.writeObject(treeString);
return newTreeID;
};
let newTree = await makeTree(head?.root, parts);
let commit = await this.makeCommit(newTree, head);
await this.writeHead(commit);
} finally {
await lock();
}
}
async treeFindObjectID(
private async treeFindObjectID(
treeID: string,
parts: string[],
type: NodeType
@ -192,9 +115,10 @@ export default class Repository {
}
}
async readTree(id: string): Promise<TreeEntry[]> {
private async readTree(id: string): Promise<TreeEntry[]> {
const tree = await this.readObject(id, true);
return tree.split("\n").map((e) => {
if (tree == undefined) throw new Error("Invalid treeID");
return tree.split("\n").filter(e => e !== "").map((e) => {
const entry = e.split(" ") as TreeEntry;
const [type] = entry;
@ -210,7 +134,7 @@ export default class Repository {
});
}
async makeCommit(treeID: string, old?: Commit) {
private async makeCommit(treeID: string, old?: Commit) {
if (!old) {
// Could be called once more than necessary, if no HEAD exists.
old = await this.readHead();
@ -221,10 +145,10 @@ export default class Repository {
return await this.writeObject(commitStr);
}
async readCommit(id: string): Promise<Commit> {
if (!(await this.hasObject(id))) throw new Error(`Commit with id ${id} not found!`);
private async readCommit(id: string): Promise<Commit> {
const commitStr = await this.readObject(id, true);
if (!commitStr)
throw new Error(`Commit with id ${id} not found!`);
let commit: Commit = { id } as any;
for (const entry of commitStr.split("\n")) {
@ -236,8 +160,10 @@ export default class Repository {
break;
case "before": // TODO: Simple validity checks
commit.before = value;
break;
case "date":
commit.date = new Date(value);
break;
}
}
@ -248,18 +174,177 @@ export default class Repository {
return commit;
}
async readHead(): Promise<Commit | undefined> {
private async readHead(): Promise<Commit | undefined> {
if (!(await this.#store.has("HEAD"))) return undefined;
const head = new TextDecoder().decode(await this.#store.get("HEAD"));
return this.readCommit(head);
}
async writeHead(commitID: string): Promise<void> {
private async writeHead(commitID: string): Promise<void> {
await this.#store.set("HEAD", new TextEncoder().encode(commitID));
}
//#endregion
//#region public
async clean() {
// TODO: Cleanup broken things
}
async readdir(path: string): Promise<TreeEntry[]> {
const parts = this.splitPath(path);
const head = await this.readHead();
if (!head) return [];
const treeID = await this.treeFindObjectID(head.root, parts, "tree");
if (!treeID) return [];
return this.readTree(treeID);
}
async read(path: string): Promise<Uint8Array | undefined> {
const parts = this.splitPath(path);
const head = await this.readHead();
if (!head) return undefined;
const objectID = await this.treeFindObjectID(head.root, parts, "blob");
if (!objectID) return undefined;
return this.readObject(objectID);
}
async readByID(id: string) {
return this.readObject(id);
}
async log(path: string): Promise<any[]> {
const parts = this.splitPath(path);
const head = await this.readHead();
if (!head) return [];
let currObjectID = await this.treeFindObjectID(head.root, parts, "blob");
let history: NodeLog[] = [];
if (currObjectID) {
history.push({
id: currObjectID,
commit: head
})
}
let currCommit = head.before;
while (currCommit !== undefined) {
try {
let commit = await this.readCommit(currCommit);
let res = await this.treeFindObjectID(commit.root, parts, "blob");
if (res !== currObjectID) {
history.push({
id: res,
commit,
});
}
currObjectID = res;
currCommit = commit.before;
} catch (err) {
break;
}
}
return history;
}
async delete(path: string) {
return this.write(path, undefined);
}
async write(path: string, data: Uint8Array | undefined) {
const parts = this.splitPath(path);
let objectID: string | undefined = undefined;
if (data) {
objectID = await this.writeObject(data);
}
const lock = await this.#store.getLock();
//TODO: Improve need of locking.
try {
const head = await this.readHead();
const makeTree = async (treeID: string | undefined, parts: string[]) => {
let tree: TreeEntry[];
if (treeID) {
tree = await this.readTree(treeID);
} else {
tree = [];
}
const current = parts[0];
let existing = tree.findIndex(([, , name]) => name == current);
let entry: TreeEntry | undefined = undefined;
if (parts.length == 1) {
if (objectID) {
entry = ["blob", objectID, current];
}
} else {
let newTreeID = await makeTree(existing >= 0 ? tree[existing][1] : undefined, parts.slice(1));
if (newTreeID) {
entry = [
"tree",
newTreeID,
current,
];
}
}
if (existing >= 0) {
let ex = tree[existing];
if (parts.length == 1 && ex[0] == "tree")
throw new Error("This change would overwrite a folder!");
if (entry)
tree[existing] = entry;
else {
tree = [...tree.slice(0, existing), ...tree.slice(existing + 1)];
}
} else {
if (entry)
tree.push(entry);
}
if (tree.length > 0) {
let treeString = tree.map(([type, hash, name]) => `${type} ${hash} ${name}`).join("\n");
let newTreeID = await this.writeObject(treeString);
return newTreeID;
} else {
return undefined;
}
};
let newTree = await makeTree(head?.root, parts);
if (!newTree) { //TODO: Is this what i want?
newTree = await this.writeObject("");
}
let commit = await this.makeCommit(newTree, head);
await this.writeHead(commit);
} finally {
await lock();
}
}
async close() {
if (this.#store.close) return this.#store?.close();
}
//#endregion
}

169
src/test.ts
View File

@ -1,27 +1,164 @@
import Repository from "./repo";
import Repository, { IDataStore } from "./repo";
import FSDataStore from "./datasources/fs";
import * as Fs from "fs";
import { expect } from "chai";
import * as Crypto from "crypto";
const t = (t: string) => new TextEncoder().encode(t);
const td = (t: Uint8Array | undefined) => new TextDecoder().decode(t);
const exists = (path: string) => Fs.promises.access(path).then(() => true).catch(() => false)
async function test() {
await Fs.promises.rm("./testrepo", { recursive: true, force: true });
const ds = new FSDataStore("./testrepo");
const rep = new Repository(ds);
let test = 0;
let repoPath = "";
console.log(new TextDecoder().decode(await rep.read("hi.txt")));
beforeEach(async () => {
repoPath = "./testrepo_" + test;
console.log(" \tRunning at repo:", repoPath);
await Fs.promises.rm(repoPath, { recursive: true, force: true });
})
await Promise.all([
rep.write("hi.txt", t("Hallo Welt: " + new Date().toLocaleString())),
rep.write("hi_hallo.txt", t("Hallo Welt: " + new Date().toLocaleString())),
rep.write("hi_hallo3.txt", t("Hallo Welt: " + new Date().toLocaleString())),
]);
afterEach(async () => {
test++;
})
console.log(new TextDecoder().decode(await rep.read("hi.txt")));
describe("NodeJS DataStore", () => {
it("should throw error when creating new instance", () => {
const ds = new FSDataStore(repoPath);
expect(ds).to.not.be.undefined;
})
console.log((await rep.readdir("/"))?.map((e) => e[2]));
it("should return undefined on read on non-existant field", async () => {
const ds = new FSDataStore(repoPath);
let val = await ds.get("test");
expect(val).to.be.undefined;
})
await rep.close();
}
it("should not fail when writing a new key and create the necessary files", async () => {
const ds = new FSDataStore(repoPath);
const text = "Hallo Welt";
await ds.set("test", t(text));
test();
let ex = await exists(repoPath + "/test");
expect(ex).to.be.true;
let file_cont = await Fs.promises.readFile(repoPath + "/test", "utf-8");
expect(file_cont).to.equal(text);
})
it("has should return false on non-existing field", async () => {
const ds = new FSDataStore(repoPath);
expect(await ds.has("test")).to.be.false;
})
it("has should return true on existing field", async () => {
const ds = new FSDataStore(repoPath);
const cont = t("Hallo Welt");
await ds.set("test", cont);
let ex = await exists(repoPath + "/test");
expect(ex).to.be.true;
})
it("get should return value of existing field", async () => {
const ds = new FSDataStore(repoPath);
const text = "Hallo Welt";
await ds.set("test", t(text));
expect(td(await ds.get("test"))).to.equal(text);
})
})
describe("Basic Repo functions", () => {
let ds: IDataStore = new FSDataStore(repoPath);
beforeEach(() => {
ds = new FSDataStore(repoPath);
})
it("should not throw error when creating new instance", () => {
const repo = new Repository(ds)
expect(repo).to.not.be.undefined;
})
it("should return an empty list on an empty repository", async () => {
const repo = new Repository(ds)
const list = await repo.readdir("/");
expect(list).to.be.an("array");
expect(list.length).to.equal(0)
})
it("should return undefined on non-existant file", async () => {
const repo = new Repository(ds)
let res = await repo.read("test");
expect(res).to.be.undefined;
})
it("should be possible to write data", async () => {
const testData = "Hallo Welt";
const repo = new Repository(ds)
await repo.write("test", t(testData));
let res = await Fs.promises.readdir(repoPath + "/objects");
expect(res.length).to.be.equal(3);
expect(await exists(repoPath + "/HEAD")).to.be.true;
})
it("should be possible to write nested data", async () => {
const testData = "Hallo Welt";
const repo = new Repository(ds)
await repo.write("test/hallo-welt", t(testData));
let res = await Fs.promises.readdir(repoPath + "/objects");
expect(res.length).to.be.equal(4);
expect(await exists(repoPath + "/HEAD")).to.be.true;
})
it("readdir should return the given entries", async () => {
const testData = "Hallo Welt";
const repo = new Repository(ds)
await repo.write("test/hallo-welt", t(testData));
await repo.write("tests", t(testData));
const res = await repo.readdir("/")
expect(res.length).to.be.equal(2);
// expect(res).to.include.
})
it("should be possible to read back written data", async () => {
const testData = "Hallo Welt";
const repo = new Repository(ds)
await repo.write("test", t(testData));
const res = await repo.read("test");
expect(td(res)).to.equal(testData);
})
it("should be possible to delete entries", async () => {
const testData = "Hallo Welt";
const repo = new Repository(ds)
await repo.write("test", t(testData));
const res = await repo.read("test");
expect(td(res)).to.equal(testData);
await repo.delete("test");
const res2 = await repo.read("test");
expect(res2).to.be.undefined;
})
it("should be possible to get a list of all versions of a file", async () => {
const testData = "Hallo Welt";
const repo = new Repository(ds)
await repo.write("test", t(testData));
await repo.write("test", t(testData + 1));
await repo.write("test", t(testData + 2));
await repo.write("test", t(testData + 3));
await repo.write("test", t(testData + 4));
const res = await repo.log("test");
expect(res).to.not.be.undefined;
expect(res.length).to.equal(5);
})
})