From 4943d8790dd33c11a5256c9c1f01e33e7d544b2b Mon Sep 17 00:00:00 2001 From: Fabian Stamm Date: Mon, 3 Jul 2023 18:06:48 +0200 Subject: [PATCH] Things --- package.json | 4 +- pnpm-lock.yaml | 12 ++ src/repo.ts | 570 ++++++++++++++++++++++++++++++++++++++----------- src/repo2.ts | 25 +++ src/sync.ts | 51 ++++- 5 files changed, 536 insertions(+), 126 deletions(-) create mode 100644 src/repo2.ts diff --git a/package.json b/package.json index 3959595..76ba831 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "license": "ISC", "devDependencies": { "@types/chai": "^4.2.21", + "@types/lodash": "^4.14.178", "@types/mocha": "^9.0.0", "@types/node": "^16.4.13", "chai": "^4.3.4", @@ -22,6 +23,7 @@ "typescript": "^4.3.5" }, "dependencies": { - "jssha": "^3.2.0" + "jssha": "^3.2.0", + "lodash": "^4.17.21" } } \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 49a81ed..efb3639 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2,10 +2,12 @@ lockfileVersion: 5.3 specifiers: '@types/chai': ^4.2.21 + '@types/lodash': ^4.14.178 '@types/mocha': ^9.0.0 '@types/node': ^16.4.13 chai: ^4.3.4 jssha: ^3.2.0 + lodash: ^4.17.21 mocha: ^9.0.3 nodemon: ^2.0.12 ts-node: ^10.2.0 @@ -13,9 +15,11 @@ specifiers: dependencies: jssha: 3.2.0 + lodash: 4.17.21 devDependencies: '@types/chai': 4.2.21 + '@types/lodash': 4.14.178 '@types/mocha': 9.0.0 '@types/node': 16.4.13 chai: 4.3.4 @@ -70,6 +74,10 @@ packages: resolution: {integrity: sha512-yd+9qKmJxm496BOV9CMNaey8TWsikaZOwMRwPHQIjcOJM9oV+fi9ZMNw3JsVnbEEbo2gRTDnGEBv8pjyn67hNg==} dev: true + /@types/lodash/4.14.178: + resolution: {integrity: sha512-0d5Wd09ItQWH1qFbEyQ7oTQ3GZrMfth5JkbN3EvTKLXcHLRDSXeLnlvlOn0wvxVIwK5o2M8JzP/OWz7T3NRsbw==} + dev: true + /@types/mocha/9.0.0: resolution: {integrity: sha512-scN0hAWyLVAvLR9AyW7HoFF5sJZglyBsbPuHO4fv7JRvfmPBMfp1ozWqOf/e4wwPNxezBZXRfWzMb6iFLgEVRA==} dev: true @@ -706,6 +714,10 @@ packages: p-locate: 5.0.0 dev: true + /lodash/4.17.21: + resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + dev: false + /log-symbols/4.1.0: resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==} engines: {node: '>=10'} diff --git a/src/repo.ts b/src/repo.ts index 176d67b..b640b2b 100644 --- a/src/repo.ts +++ b/src/repo.ts @@ -11,15 +11,14 @@ export interface IDataStore { } export type NodeType = "tree" | "blob"; -export type NodeHash = string; +export type ObjectID = string; export type NodeFilename = string; -export type TreeEntry = [NodeType, NodeHash, NodeFilename]; +export type TreeEntry = [NodeType, ObjectID, NodeFilename]; -export type Commit = { - id: string; +export type ICommit = { root: string; before: string; - merge: string; + merge?: string; date: Date; }; @@ -49,8 +48,57 @@ export const ObjectTypes = { } export type ObjectTypeNames = keyof typeof ObjectTypes; +export type TypedObjectData = { + type: ObjectTypeNames, + data: Uint8Array +} export default class Repository { + //#region static + + static sha1(data: Uint8Array) { + const s = new SHA("SHA-1", "UINT8ARRAY"); + s.update(data); + return s.getHash("HEX"); + } + + static sha256(data: Uint8Array) { + const s = new SHA("SHA3-256", "UINT8ARRAY"); + s.update(data); + return s.getHash("HEX"); + } + + static generateObjectID(data: Uint8Array) { + return this.sha1(data); + } + + static parseObject(data: Uint8Array | undefined): TypedObjectData | undefined { + if (!data) + return undefined; + + let type = new TextDecoder().decode(data.slice(0, 4)) + return { + type: type as ObjectTypeNames, + data: data.slice(5) + } + } + + static buildObject(data: Uint8Array | string, type: ObjectTypeNames): ObjectTuple { + if (typeof data == "string") { + data = new TextEncoder().encode(data); + } + const objectID = this.generateObjectID(data); + + let merged = new Uint8Array(5 + data.length); + merged.set(ObjectTypes[type], 0) + merged.set(data, 5); + + return [objectID, merged] + } + + //#endregion + + //#region local variables #store: IDataStore; //#endregion @@ -61,17 +109,7 @@ export default class Repository { //#region private - private sha1(data: Uint8Array) { - const s = new SHA("SHA-1", "UINT8ARRAY"); - s.update(data); - return s.getHash("HEX"); - } - private sha256(data: Uint8Array) { - const s = new SHA("SHA3-256", "UINT8ARRAY"); - s.update(data); - return s.getHash("HEX"); - } private splitPath(path: string) { const resolved = Path.resolve(path).slice(1); @@ -105,12 +143,6 @@ export default class Repository { } } - private async readTree(id: string): Promise { - const tree = await this.readObject(id, true); - if (tree == undefined) throw new Error("Invalid treeID"); - return this.parseTree(tree); - } - private async makeCommit(treeID: string, old?: Commit, merge?: Commit) { if (!old) { // Could be called once more than necessary, if no HEAD exists. @@ -145,7 +177,7 @@ export default class Repository { if (typeof data == "string") { data = new TextEncoder().encode(data); } - const objectID = this.sha1(data); + const objectID = Repository.generateObjectID(data); let merged = new Uint8Array(5 + data.length); merged.set(ObjectTypes[type], 0) @@ -165,16 +197,11 @@ export default class Repository { public async readObjectTyped(id: String) { let data = await this.#store.get("objects/" + id); - if (!data) - return undefined; - - let type = new TextDecoder().decode(data.slice(0, 4)) - return { - type: type as ObjectTypeNames, - data: data.slice(5) - } + return Repository.parseObject(data); } + + public async readObject(id: string, string: true): Promise; public async readObject(id: string, string?: false): Promise; public async readObject(id: string, string = false): Promise { @@ -204,52 +231,21 @@ export default class Repository { return type == "comm"; } - parseTree(treeStr: string | Uint8Array): TreeEntry[] { - if (typeof treeStr !== "string") - treeStr = new TextDecoder().decode(treeStr); - return treeStr.split("\n").filter(e => e !== "").map((e) => { - const entry = e.split(" ") as TreeEntry; - const [type] = entry; - - switch (type) { - case "blob": - case "tree": - break; - default: - throw new Error("Invalid tree type."); //Might be a newer version or so - } - - return entry; - }); + private getVirtualRepo() { + return new VirtualRepo(this); } - parseCommit(id: string, commitStr: string | Uint8Array,) { - if (typeof commitStr !== "string") - commitStr = new TextDecoder().decode(commitStr); - - let commit: Commit = { id } as any; - for (const entry of commitStr.split("\n")) { - const [type] = entry.split(" ", 1); - const value = entry.slice(type.length + 1); - switch (type) { - case "tree": // TODO: Simple validity checks - commit.root = value; - break; - case "before": // TODO: Simple validity checks - commit.before = value; - break; - case "merge": // TODO: Simple validity checks - commit.merge = value; - break; - case "date": - commit.date = new Date(value); - break; - } - } - - return commit; + async readTree(id: string): Promise { + return Tree.load(this.getVirtualRepo(), id); } + async readBlob(id: string): Promise { + return Blob.load(this.getVirtualRepo(), id); + } + + + + async readCommit(id: string): Promise { const commitStr = await this.readObject(id, true); if (!commitStr) @@ -264,6 +260,13 @@ export default class Repository { return commit; } + async getHead() { + if (!(await this.#store.has("HEAD"))) return undefined; + const head = new TextDecoder().decode(await this.#store.get("HEAD")); + + return head; + } + async readHead(): Promise { if (!(await this.#store.has("HEAD"))) return undefined; const head = new TextDecoder().decode(await this.#store.get("HEAD")); @@ -272,8 +275,36 @@ export default class Repository { } async mergeCommits(commit1: string, commit2: string): Promise { - throw new Error("WIP"); - // let newCommit = this.makeCommit() + // throw new Error("WIP"); + // Find common history (if there). Diff changes. + + const getCommitPath = async (commitID: string) => { + let commits: string[] = []; + let com = await this.readCommit(commitID) + if (com.before) { + commits = [...await getCommitPath(com.before), com.before] + } + + if (com.merge) { //TODO: Might cause duplicates? + commits = [...await getCommitPath(com.merge), com.merge]; + } + + return commits; + } + + let commits1 = await getCommitPath(commit1); + let commitSet2 = new Set(await getCommitPath(commit2)); + + let common = commits1.find(commit => commitSet2.has(commit)); + if (!common) { + // Complete merge of trees + } + + let newTree = + + + + let newCommit = this.makeCommit() } @@ -288,7 +319,7 @@ export default class Repository { while (current.before || (till && current.id == till)) { current = await this.readCommit(current.before); if (!current) - throw new Error("Repository sems damaged! Can't read commit!"); + throw new Error("Repository seems damaged! Can't read commit!"); log.push(current) } @@ -374,64 +405,73 @@ export default class Repository { objectID = await this.writeObject(data, "blob"); } + const vr = this.getVirtualRepo() + + let blob = data ? Blob.create(vr, data) : undefined; + const lock = await this.#store.getLock(); //TODO: Improve need of locking. try { - const head = await this.readHead(); + const head = await this.getHead(); + let com: Commit; + if(head) + com = Commit.load(vr, head); + else + com = Commit.create(vr); - const makeTree = async (treeID: string | undefined, parts: string[]) => { - let tree: TreeEntry[]; - if (treeID) { - tree = await this.readTree(treeID); - } else { - tree = []; - } + // const makeTree = async (treeID: string | undefined, parts: string[]) => { + // let tree: TreeEntry[]; + // if (treeID) { + // tree = await this.readTree(treeID); + // } else { + // tree = []; + // } - const current = parts[0]; + // const current = parts[0]; - let existing = tree.findIndex(([, , name]) => name == current); + // let existing = tree.findIndex(([, , name]) => name == current); - let entry: TreeEntry | undefined = undefined; + // let entry: TreeEntry | undefined = undefined; - if (parts.length == 1) { - if (objectID) { - entry = ["blob", objectID, current]; - } - } else { - let newTreeID = await makeTree(existing >= 0 ? tree[existing][1] : undefined, parts.slice(1)); - if (newTreeID) { - entry = [ - "tree", - newTreeID, - current, - ]; - } - } + // if (parts.length == 1) { + // if (objectID) { + // entry = ["blob", objectID, current]; + // } + // } else { + // let newTreeID = await makeTree(existing >= 0 ? tree[existing][1] : undefined, parts.slice(1)); + // if (newTreeID) { + // entry = [ + // "tree", + // newTreeID, + // current, + // ]; + // } + // } - if (existing >= 0) { - let ex = tree[existing]; - if (parts.length == 1 && ex[0] == "tree") - throw new Error("This change would overwrite a folder!"); - if (entry) - tree[existing] = entry; - else { - tree = [...tree.slice(0, existing), ...tree.slice(existing + 1)]; - } - } else { - if (entry) - tree.push(entry); - } + // if (existing >= 0) { + // let ex = tree[existing]; + // if (parts.length == 1 && ex[0] == "tree") + // throw new Error("This change would overwrite a folder!"); + // if (entry) + // tree[existing] = entry; + // else { + // tree = [...tree.slice(0, existing), ...tree.slice(existing + 1)]; + // } + // } else { + // if (entry) + // tree.push(entry); + // } - if (tree.length > 0) { - let treeString = tree.map(([type, hash, name]) => `${type} ${hash} ${name}`).join("\n"); + // if (tree.length > 0) { + // let treeString = tree.map(([type, hash, name]) => `${type} ${hash} ${name}`).join("\n"); - let newTreeID = await this.writeObject(treeString, "tree"); + // let newTreeID = await this.writeObject(treeString, "tree"); - return newTreeID; - } else { - return undefined; - } - }; + // return newTreeID; + // } else { + // return undefined; + // } + // }; let newTree = await makeTree(head?.root, parts); if (!newTree) { //TODO: Is this what i want? @@ -448,5 +488,293 @@ export default class Repository { if (this.#store.close) return this.#store?.close(); } + async applyVirtualCommit() { + //TODO: Maybe this?? + } + //#endregion +} + +class VirtualRepo { + #virtualObjects = new Map(); + constructor(protected repo: Repository) { } + + async getObject(id: string): Promise { + let local = this.#virtualObjects.get(id); + if (local) + return Repository.parseObject(local); + + return this.repo.readObjectTyped(id); + } + + async putObject(data: Uint8Array | string, type: ObjectTypeNames): Promise { + let [id, d] = Repository.buildObject(data, type); + this.#virtualObjects.set(id, d); + return id; + } + + async write() { + //TODO: Implement or use applyVirtualCommit of repo + } +} + +type ObjectTuple = [string, Uint8Array]; + +abstract class RepoObject { + public abstract getObject(): Promise<[ObjectID, ObjectID[]] | null>; + public abstract freeze(): void; +} + +class Blob extends RepoObject { + #id: ObjectID; + + get id(): ObjectID { + return this.#id; + } + + constructor(private vr: VirtualRepo, id: string) { + super(); + this.#id = id; + } + + async getObject() { + this.freeze(); + return [this.#id, []] as [ObjectID, ObjectID[]]; + } + + freeze( ) { + Object.freeze(this); + } + + static async load(vr: VirtualRepo, id: string) { + return new Blob(vr, id); + } + + static async create(vr: VirtualRepo, data: Uint8Array | string) { + let id = await vr.putObject(data, "blob") + return new Blob(vr, id); + } +} + +class Tree extends RepoObject { + trees: Map = new Map(); + blobs: Map = new Map(); + + constructor(private vr: VirtualRepo) { + super(); + } + + static async load(vr: VirtualRepo, id: string) { + const res = await vr.getObject(id); + if (!res) + throw new Error("Invalid TreeID") + if (res.type !== "tree") + throw new Error("This item is not a tree!"); + + return this.parseTree(vr, res.data); + } + + static async create(vr: VirtualRepo) { + return new Tree(vr); + } + + setTree(name: string, tree: Tree | null) { + if (this.blobs.has(name)) + this.blobs.delete(name); // TODO: Think if error or silent overwrite + if (!tree) + this.trees.delete(name); + else + this.trees.set(name, tree); + } + + setBlob(name: string, blob: Blob | null) { + if (this.trees.has(name)) + this.trees.delete(name); // TODO: Think if error or silent overwrite + if (!blob) + return this.blobs.delete(name); + else + this.blobs.set(name, blob); + } + + freeze() { + Object.freeze(this.blobs) + Object.freeze(this.trees) + Object.freeze(this); + } + + async getObject(): Promise<[string, string[]] | null> { + this.freeze(); + if (this.trees.size <= 0 && this.blobs.size <= 0) + return null; + + let trees = (await Promise.all([...this.trees.entries()].sort(([k1], [k2]) => { + if (k1 < k2) + return -1; + if (k2 > k2) + return 1; + return 0; + }).map(async ([name, element]) => { + let object = await element.getObject(); + if (!object) + return undefined; + return { + name, + id: object[0], + dependencies: object[1] + } + }))).filter(e => !!e) as { name: string, id: string, dependencies: string[] }[]; + + let blobs = (await Promise.all([...this.blobs.entries()].sort(([k1], [k2]) => { + if (k1 < k2) + return -1; + if (k2 > k2) + return 1; + return 0; + }).map(async ([name, element]) => { + let object = await element.getObject(); + if (!object) + return undefined; + return { + name, + id: object[0], + dependencies: object[1] + } + }))).filter(e => !!e) as { name: string, id: string, dependencies: string[] }[]; + + + let dependencies = [ + ...trees.map(e => e?.id), + ...trees.map(e => e?.dependencies).flat(1), + ...blobs.map(e => e?.id), + ...blobs.map(e => e?.dependencies).flat(1), + ]; + + + let rows: string[] = []; + trees.forEach((tree) => { + rows.push(`tree ${tree.id} ${tree.name}`); + }) + + blobs.forEach((blob) => { + rows.push(`blob ${blob.id} ${blob.name}`); + }) + + let content = rows.join("\n"); + let newID = await this.vr.putObject(content, "tree"); + + + //TODO: + // 1. Get Subtrees and their new IDs + // 2. Emit Subtrees/Blobs and new version of itself + + return [newID, dependencies]; + } + + static async parseTree(vr: VirtualRepo, treeStr: string | Uint8Array) { + if (typeof treeStr !== "string") + treeStr = new TextDecoder().decode(treeStr); + + let tree = new Tree(vr); + + await Promise.all(treeStr.split("\n").filter(e => e !== "").map(async (e) => { + const entry = e.split(" ") as TreeEntry; + const [type, hash, filename] = entry; + + switch (type) { + case "blob": + tree.blobs.set(filename, await Blob.load(vr, hash)); + break; + case "tree": + tree.trees.set(filename, await Tree.load(vr, hash)); + break; + default: + throw new Error("Invalid tree type."); //Might be a newer version or so + } + + return entry; + })); + + return tree; + } +} + +class Commit extends RepoObject { + root?: Tree = undefined; + before?: string = undefined; + date?: Date = undefined; + merge?: string; + + constructor(private vr: VirtualRepo) { + super(); + } + + + freeze() { + Object.freeze(this.date); + Object.freeze(this); + } + + async getObject() { + if (!this.root) + throw new Error("This commit has no root!"); + + if (!this.date) + this.date = new Date(); + + this.freeze() + const rootData = await this.root.getObject(); + + if (!rootData) //TODO: Might actually happen right now, when the last file is deleted + throw new Error("Invalid tree!") + + const [treeID, dependencies] = rootData; + + let commitStr = + `tree ${treeID}\ndate ${new Date().toISOString()}\n`; + if (this.before) { + commitStr += `before ${this.before}\n`; + if (this.merge) { + commitStr += `merge ${this.merge}\n`; + } + } + + let commitID = await this.vr.putObject(commitStr, "comm"); + + return [commitID, [...dependencies, treeID]] as [string, string[]]; + } + + static load(vr: VirtualRepo, id: string) { + let res = await vr.getObject(id); + if (!res) + throw new Error("Invalid ID"); + + let commitStr = new TextDecoder().decode(res.data); + + let commit = new Commit(vr); + + for (const entry of commitStr.split("\n")) { + const [type] = entry.split(" ", 1); + const value = entry.slice(type.length + 1); + switch (type) { + case "tree": // TODO: Simple validity checks + commit.root = await Tree.load(vr, value); + break; + case "before": // TODO: Simple validity checks + commit.before = value; + break; + case "merge": // TODO: Simple validity checks + commit.merge = value; + break; + case "date": + commit.date = new Date(value); + break; + } + } + + return commit; + } + + static create(vr: VirtualRepo) { + return new Commit(vr); + } } \ No newline at end of file diff --git a/src/repo2.ts b/src/repo2.ts new file mode 100644 index 0000000..75b8eec --- /dev/null +++ b/src/repo2.ts @@ -0,0 +1,25 @@ +import * as fs from "fs"; + +import { compose, multiply, prop, sumBy, map, pipe, pluck, filter, lt } from 'lodash/fp'; + + +class Object { + +} + +class Blob { + +} + +class Tree { + +} + +class Commit { + +} + + +class Repo { + +} \ No newline at end of file diff --git a/src/sync.ts b/src/sync.ts index a3bc237..5c22611 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -6,7 +6,7 @@ import { ServiceProvider as ServiceProviderClient } from "./com/service_client"; import { ServiceProvider as ServiceProviderServer } from "./com/service_server"; import { PullObjectRequest, PullObjectResponse, PushObjectRequest, PushObjectResponse, Sync as SyncClient, SyncInitRequest, SyncInitResponse } from "./com/sync_client"; import { Sync as SyncServer } from "./com/sync_server"; -import Repository, { ObjectTypes } from "./repo"; +import Repository, { Commit, ObjectTypes, TreeEntry } from "./repo"; if (typeof btoa === 'undefined') { global.btoa = function (str) { @@ -189,10 +189,53 @@ export async function startSyncClient(stream: ISimpleStream, repo: Repository) { let localHead = await repo.readHead() if (localHead) { // If there is nothing to push, don't push - // Find matching point - let match = undefined; + let toCheck: Commit[] = [localHead]; - if (!syncResponse.remoteCommit) { } + let commit: Commit; + while(toCheck.length > 0) { + commit = toCheck.shift() as Commit; + if(commit.id != syncResponse.remoteCommit) { //Not yet at remote commit. Wait + commitsToPush.add(commit.id); + if(commit.before && !commitsToPush.has(commit.before)) // A commit could already be checked by the other part of a merge + toCheck.push(await repo.readCommit(commit.before)); + + if(commit.merge && !commitsToPush.has(commit.merge)) + toCheck.push(await repo.readCommit(commit.merge)); + } + } + + const traverseTree =async (current: TreeEntry[]) => { + for(const [type, hash, name] of current) { + objectsToPush.add(hash); + if(type =="tree") { + let tree = await repo.readTree(hash); + await traverseTree(tree); + } + } + } + + for(const commitId of commitsToPush) { + let commit = await repo.readCommit(commitId); + let rootId = commit.root + + objectsToPush.add(rootId); + + if(objectsToPush.has(rootId)) { + objectsToPush.add(rootId); + let root = await repo.readTree(rootId); + await traverseTree(root); + } + + } + } + + for(const objId of objectsToPush) { + let obj = await repo.readObjectTyped(objId) + await service.PushObject({ + id: objId, + data: obj?.data as Uint8Array, + type: obj?.type as string + }); } }