SimpleVSC/src/repo.ts

780 lines
20 KiB
TypeScript

import SHA from "jssha";
import Path from "./helper/path";
export interface IDataStore {
get(key: string): Promise<Uint8Array | undefined>;
set(key: string, data: Uint8Array): Promise<void>;
has(key: string): Promise<boolean>;
close?: () => Promise<void>;
getLock: () => Promise<() => Promise<void>>;
}
export type NodeType = "tree" | "blob";
export type ObjectID = string;
export type NodeFilename = string;
export type TreeEntry = [NodeType, ObjectID, NodeFilename];
export type ICommit = {
root: string;
before: string;
merge?: string;
date: Date;
};
export type NodeLog = {
/**
* ObjectID of the data. Can be undefined if element was deleted!
*/
id: string | undefined;
commit: Commit
}
export type ISyncTransferStream = {
on(type: "data", func: (data: Uint8Array) => void): void;
send(data: Uint8Array): void;
}
// TODOs:
// - HEAD locks
// - HEAD/Tree Cache
// - Remote synchronisation
// - Add DataStore Locking for access from multiple sources
export const ObjectTypes = {
blob: new Uint8Array([98, 108, 111, 98, 10]),
tree: new Uint8Array([116, 114, 101, 101, 10]),
comm: new Uint8Array([99, 111, 109, 109, 10]),
}
export type ObjectTypeNames = keyof typeof ObjectTypes;
export type TypedObjectData = {
type: ObjectTypeNames,
data: Uint8Array
}
export default class Repository {
//#region static
static sha1(data: Uint8Array) {
const s = new SHA("SHA-1", "UINT8ARRAY");
s.update(data);
return s.getHash("HEX");
}
static sha256(data: Uint8Array) {
const s = new SHA("SHA3-256", "UINT8ARRAY");
s.update(data);
return s.getHash("HEX");
}
static generateObjectID(data: Uint8Array) {
return this.sha1(data);
}
static parseObject(data: Uint8Array | undefined): TypedObjectData | undefined {
if (!data)
return undefined;
let type = new TextDecoder().decode(data.slice(0, 4))
return {
type: type as ObjectTypeNames,
data: data.slice(5)
}
}
static buildObject(data: Uint8Array | string, type: ObjectTypeNames): ObjectTuple {
if (typeof data == "string") {
data = new TextEncoder().encode(data);
}
const objectID = this.generateObjectID(data);
let merged = new Uint8Array(5 + data.length);
merged.set(ObjectTypes[type], 0)
merged.set(data, 5);
return [objectID, merged]
}
//#endregion
//#region local variables
#store: IDataStore;
//#endregion
constructor(store: IDataStore) {
this.#store = store;
}
//#region private
private splitPath(path: string) {
const resolved = Path.resolve(path).slice(1);
if (resolved == "") return [];
return resolved.split(Path.sep);
}
private async treeFindObjectID(
treeID: string,
parts: string[],
type: NodeType
): Promise<string | undefined> {
if (parts.length == 0) {
if (type == "tree") return treeID;
else throw new Error("Cannot open root as file!");
}
const tree = await this.readTree(treeID);
const current = parts[0];
let entry = tree.find(([type, id, name]) => name == current);
if (!entry) {
return undefined;
}
if (parts.length == 1) {
if (entry[0] != type) throw new Error("Expected 'blob' found 'tree'!");
return entry[1];
} else {
if (entry[0] != "tree") throw new Error("Expected 'tree' found 'blob'!");
return this.treeFindObjectID(entry[1], parts.splice(1), type);
}
}
private async makeCommit(treeID: string, old?: Commit, merge?: Commit) {
if (!old) {
// Could be called once more than necessary, if no HEAD exists.
old = await this.readHead();
}
let commitStr =
`tree ${treeID}\ndate ${new Date().toISOString()}\n`;
if (old) {
commitStr += `before ${old?.id}\n`;
if (merge) {
commitStr += `merge ${old?.id}\n`;
}
}
return await this.writeObject(commitStr, "comm");
}
//#endregion
//#region public
async clean() {
// TODO: Cleanup broken things
}
async writeHead(commitID: string): Promise<void> {
await this.#store.set("HEAD", new TextEncoder().encode(commitID));
}
public async writeObject(data: Uint8Array | string, type: ObjectTypeNames): Promise<string> {
if (typeof data == "string") {
data = new TextEncoder().encode(data);
}
const objectID = Repository.generateObjectID(data);
let merged = new Uint8Array(5 + data.length);
merged.set(ObjectTypes[type], 0)
merged.set(data, 5);
await this.#store.set("objects/" + objectID, merged);
return objectID;
}
public async hasObject(id: string): Promise<boolean> {
return this.#store.has("objects/" + id);
}
public async readObjectRaw(id: string) {
return await this.#store.get("objects/" + id);
}
public async readObjectTyped(id: String) {
let data = await this.#store.get("objects/" + id);
return Repository.parseObject(data);
}
public async readObject(id: string, string: true): Promise<string | undefined>;
public async readObject(id: string, string?: false): Promise<Uint8Array | undefined>;
public async readObject(id: string, string = false): Promise<Uint8Array | string | undefined> {
const res = await this.readObjectTyped(id);
if (!res)
return undefined;
const { data } = res;
if (string) {
return new TextDecoder().decode(data);
} else {
return data;
}
}
public async readObjectType(id: string): Promise<ObjectTypeNames | undefined> {
const res = await this.readObjectTyped(id);
if (!res)
return undefined;
return res.type;
}
async hasCommit(id: string) {
let type = await this.readObjectType(id)
return type == "comm";
}
private getVirtualRepo() {
return new VirtualRepo(this);
}
async readTree(id: string): Promise<Tree> {
return Tree.load(this.getVirtualRepo(), id);
}
async readBlob(id: string): Promise<Blob> {
return Blob.load(this.getVirtualRepo(), id);
}
async readCommit(id: string): Promise<Commit> {
const commitStr = await this.readObject(id, true);
if (!commitStr)
throw new Error(`Commit with id ${id} not found!`);
let commit = this.parseCommit(id, commitStr);
if (!commit.root) {
throw new Error("No tree defined in this commit!");
}
return commit;
}
async getHead() {
if (!(await this.#store.has("HEAD"))) return undefined;
const head = new TextDecoder().decode(await this.#store.get("HEAD"));
return head;
}
async readHead(): Promise<Commit | undefined> {
if (!(await this.#store.has("HEAD"))) return undefined;
const head = new TextDecoder().decode(await this.#store.get("HEAD"));
return this.readCommit(head);
}
async mergeCommits(commit1: string, commit2: string): Promise<string> {
// throw new Error("WIP");
// Find common history (if there). Diff changes.
const getCommitPath = async (commitID: string) => {
let commits: string[] = [];
let com = await this.readCommit(commitID)
if (com.before) {
commits = [...await getCommitPath(com.before), com.before]
}
if (com.merge) { //TODO: Might cause duplicates?
commits = [...await getCommitPath(com.merge), com.merge];
}
return commits;
}
let commits1 = await getCommitPath(commit1);
let commitSet2 = new Set(await getCommitPath(commit2));
let common = commits1.find(commit => commitSet2.has(commit));
if (!common) {
// Complete merge of trees
}
let newTree =
let newCommit = this.makeCommit()
}
async readCommitLog(till?: string, merges?: boolean): Promise<Commit[]> {
let head = await this.readHead();
if (!head)
return [];
let log: Commit[] = [head];
let current = head;
while (current.before || (till && current.id == till)) {
current = await this.readCommit(current.before);
if (!current)
throw new Error("Repository seems damaged! Can't read commit!");
log.push(current)
}
return log;
}
async readdir(path: string): Promise<TreeEntry[]> {
const parts = this.splitPath(path);
const head = await this.readHead();
if (!head) return [];
const treeID = await this.treeFindObjectID(head.root, parts, "tree");
if (!treeID) return [];
return this.readTree(treeID);
}
async read(path: string): Promise<Uint8Array | undefined> {
const parts = this.splitPath(path);
const head = await this.readHead();
if (!head) return undefined;
const objectID = await this.treeFindObjectID(head.root, parts, "blob");
if (!objectID) return undefined;
return this.readObject(objectID);
}
async readByID(id: string) {
return this.readObject(id);
}
async fileLog(path: string): Promise<any[]> {
const parts = this.splitPath(path);
const head = await this.readHead();
if (!head) return [];
let currObjectID = await this.treeFindObjectID(head.root, parts, "blob");
let history: NodeLog[] = [];
if (currObjectID) {
history.push({
id: currObjectID,
commit: head
})
}
let currCommit = head.before;
while (currCommit !== undefined) {
try {
let commit = await this.readCommit(currCommit);
let res = await this.treeFindObjectID(commit.root, parts, "blob");
if (res !== currObjectID) {
history.push({
id: res,
commit,
});
}
currObjectID = res;
currCommit = commit.before;
} catch (err) {
break;
}
}
return history;
}
async delete(path: string) {
return this.write(path, undefined);
}
async write(path: string, data: Uint8Array | undefined) {
const parts = this.splitPath(path);
let objectID: string | undefined = undefined;
if (data) {
objectID = await this.writeObject(data, "blob");
}
const vr = this.getVirtualRepo()
let blob = data ? Blob.create(vr, data) : undefined;
const lock = await this.#store.getLock();
//TODO: Improve need of locking.
try {
const head = await this.getHead();
let com: Commit;
if(head)
com = Commit.load(vr, head);
else
com = Commit.create(vr);
// const makeTree = async (treeID: string | undefined, parts: string[]) => {
// let tree: TreeEntry[];
// if (treeID) {
// tree = await this.readTree(treeID);
// } else {
// tree = [];
// }
// const current = parts[0];
// let existing = tree.findIndex(([, , name]) => name == current);
// let entry: TreeEntry | undefined = undefined;
// if (parts.length == 1) {
// if (objectID) {
// entry = ["blob", objectID, current];
// }
// } else {
// let newTreeID = await makeTree(existing >= 0 ? tree[existing][1] : undefined, parts.slice(1));
// if (newTreeID) {
// entry = [
// "tree",
// newTreeID,
// current,
// ];
// }
// }
// if (existing >= 0) {
// let ex = tree[existing];
// if (parts.length == 1 && ex[0] == "tree")
// throw new Error("This change would overwrite a folder!");
// if (entry)
// tree[existing] = entry;
// else {
// tree = [...tree.slice(0, existing), ...tree.slice(existing + 1)];
// }
// } else {
// if (entry)
// tree.push(entry);
// }
// if (tree.length > 0) {
// let treeString = tree.map(([type, hash, name]) => `${type} ${hash} ${name}`).join("\n");
// let newTreeID = await this.writeObject(treeString, "tree");
// return newTreeID;
// } else {
// return undefined;
// }
// };
let newTree = await makeTree(head?.root, parts);
if (!newTree) { //TODO: Is this what i want?
newTree = await this.writeObject("", "tree");
}
let commit = await this.makeCommit(newTree, head);
await this.writeHead(commit);
} finally {
await lock();
}
}
async close() {
if (this.#store.close) return this.#store?.close();
}
async applyVirtualCommit() {
//TODO: Maybe this??
}
//#endregion
}
class VirtualRepo {
#virtualObjects = new Map<string, Uint8Array>();
constructor(protected repo: Repository) { }
async getObject(id: string): Promise<TypedObjectData | undefined> {
let local = this.#virtualObjects.get(id);
if (local)
return Repository.parseObject(local);
return this.repo.readObjectTyped(id);
}
async putObject(data: Uint8Array | string, type: ObjectTypeNames): Promise<ObjectID> {
let [id, d] = Repository.buildObject(data, type);
this.#virtualObjects.set(id, d);
return id;
}
async write() {
//TODO: Implement or use applyVirtualCommit of repo
}
}
type ObjectTuple = [string, Uint8Array];
abstract class RepoObject {
public abstract getObject(): Promise<[ObjectID, ObjectID[]] | null>;
public abstract freeze(): void;
}
class Blob extends RepoObject {
#id: ObjectID;
get id(): ObjectID {
return this.#id;
}
constructor(private vr: VirtualRepo, id: string) {
super();
this.#id = id;
}
async getObject() {
this.freeze();
return [this.#id, []] as [ObjectID, ObjectID[]];
}
freeze( ) {
Object.freeze(this);
}
static async load(vr: VirtualRepo, id: string) {
return new Blob(vr, id);
}
static async create(vr: VirtualRepo, data: Uint8Array | string) {
let id = await vr.putObject(data, "blob")
return new Blob(vr, id);
}
}
class Tree extends RepoObject {
trees: Map<string, Tree> = new Map();
blobs: Map<string, Blob> = new Map();
constructor(private vr: VirtualRepo) {
super();
}
static async load(vr: VirtualRepo, id: string) {
const res = await vr.getObject(id);
if (!res)
throw new Error("Invalid TreeID")
if (res.type !== "tree")
throw new Error("This item is not a tree!");
return this.parseTree(vr, res.data);
}
static async create(vr: VirtualRepo) {
return new Tree(vr);
}
setTree(name: string, tree: Tree | null) {
if (this.blobs.has(name))
this.blobs.delete(name); // TODO: Think if error or silent overwrite
if (!tree)
this.trees.delete(name);
else
this.trees.set(name, tree);
}
setBlob(name: string, blob: Blob | null) {
if (this.trees.has(name))
this.trees.delete(name); // TODO: Think if error or silent overwrite
if (!blob)
return this.blobs.delete(name);
else
this.blobs.set(name, blob);
}
freeze() {
Object.freeze(this.blobs)
Object.freeze(this.trees)
Object.freeze(this);
}
async getObject(): Promise<[string, string[]] | null> {
this.freeze();
if (this.trees.size <= 0 && this.blobs.size <= 0)
return null;
let trees = (await Promise.all([...this.trees.entries()].sort(([k1], [k2]) => {
if (k1 < k2)
return -1;
if (k2 > k2)
return 1;
return 0;
}).map(async ([name, element]) => {
let object = await element.getObject();
if (!object)
return undefined;
return {
name,
id: object[0],
dependencies: object[1]
}
}))).filter(e => !!e) as { name: string, id: string, dependencies: string[] }[];
let blobs = (await Promise.all([...this.blobs.entries()].sort(([k1], [k2]) => {
if (k1 < k2)
return -1;
if (k2 > k2)
return 1;
return 0;
}).map(async ([name, element]) => {
let object = await element.getObject();
if (!object)
return undefined;
return {
name,
id: object[0],
dependencies: object[1]
}
}))).filter(e => !!e) as { name: string, id: string, dependencies: string[] }[];
let dependencies = [
...trees.map(e => e?.id),
...trees.map(e => e?.dependencies).flat(1),
...blobs.map(e => e?.id),
...blobs.map(e => e?.dependencies).flat(1),
];
let rows: string[] = [];
trees.forEach((tree) => {
rows.push(`tree ${tree.id} ${tree.name}`);
})
blobs.forEach((blob) => {
rows.push(`blob ${blob.id} ${blob.name}`);
})
let content = rows.join("\n");
let newID = await this.vr.putObject(content, "tree");
//TODO:
// 1. Get Subtrees and their new IDs
// 2. Emit Subtrees/Blobs and new version of itself
return [newID, dependencies];
}
static async parseTree(vr: VirtualRepo, treeStr: string | Uint8Array) {
if (typeof treeStr !== "string")
treeStr = new TextDecoder().decode(treeStr);
let tree = new Tree(vr);
await Promise.all(treeStr.split("\n").filter(e => e !== "").map(async (e) => {
const entry = e.split(" ") as TreeEntry;
const [type, hash, filename] = entry;
switch (type) {
case "blob":
tree.blobs.set(filename, await Blob.load(vr, hash));
break;
case "tree":
tree.trees.set(filename, await Tree.load(vr, hash));
break;
default:
throw new Error("Invalid tree type."); //Might be a newer version or so
}
return entry;
}));
return tree;
}
}
class Commit extends RepoObject {
root?: Tree = undefined;
before?: string = undefined;
date?: Date = undefined;
merge?: string;
constructor(private vr: VirtualRepo) {
super();
}
freeze() {
Object.freeze(this.date);
Object.freeze(this);
}
async getObject() {
if (!this.root)
throw new Error("This commit has no root!");
if (!this.date)
this.date = new Date();
this.freeze()
const rootData = await this.root.getObject();
if (!rootData) //TODO: Might actually happen right now, when the last file is deleted
throw new Error("Invalid tree!")
const [treeID, dependencies] = rootData;
let commitStr =
`tree ${treeID}\ndate ${new Date().toISOString()}\n`;
if (this.before) {
commitStr += `before ${this.before}\n`;
if (this.merge) {
commitStr += `merge ${this.merge}\n`;
}
}
let commitID = await this.vr.putObject(commitStr, "comm");
return [commitID, [...dependencies, treeID]] as [string, string[]];
}
static load(vr: VirtualRepo, id: string) {
let res = await vr.getObject(id);
if (!res)
throw new Error("Invalid ID");
let commitStr = new TextDecoder().decode(res.data);
let commit = new Commit(vr);
for (const entry of commitStr.split("\n")) {
const [type] = entry.split(" ", 1);
const value = entry.slice(type.length + 1);
switch (type) {
case "tree": // TODO: Simple validity checks
commit.root = await Tree.load(vr, value);
break;
case "before": // TODO: Simple validity checks
commit.before = value;
break;
case "merge": // TODO: Simple validity checks
commit.merge = value;
break;
case "date":
commit.date = new Date(value);
break;
}
}
return commit;
}
static create(vr: VirtualRepo) {
return new Commit(vr);
}
}