hibas123
/
SimpleVSC
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Things

This commit is contained in:
Fabian Stamm 2023-07-03 18:06:48 +02:00
parent 1a7400e38c
commit 4943d8790d
5 changed files with 536 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
package.json
View File

@ -13,6 +13,7 @@
"license": "ISC",
"devDependencies": {
"@types/chai": "^4.2.21",
"@types/lodash": "^4.14.178",
"@types/mocha": "^9.0.0",
"@types/node": "^16.4.13",
"chai": "^4.3.4",
@ -22,6 +23,7 @@
"typescript": "^4.3.5"
},
"dependencies": {
"jssha": "^3.2.0"
"jssha": "^3.2.0",
"lodash": "^4.17.21"
}
}

12
pnpm-lock.yaml
View File

@ -2,10 +2,12 @@ lockfileVersion: 5.3
specifiers:
'@types/chai': ^4.2.21
'@types/lodash': ^4.14.178
'@types/mocha': ^9.0.0
'@types/node': ^16.4.13
chai: ^4.3.4
jssha: ^3.2.0
lodash: ^4.17.21
mocha: ^9.0.3
nodemon: ^2.0.12
ts-node: ^10.2.0
@ -13,9 +15,11 @@ specifiers:
dependencies:
jssha: 3.2.0
lodash: 4.17.21
devDependencies:
'@types/chai': 4.2.21
'@types/lodash': 4.14.178
'@types/mocha': 9.0.0
'@types/node': 16.4.13
chai: 4.3.4
@ -70,6 +74,10 @@ packages:
resolution: {integrity: sha512-yd+9qKmJxm496BOV9CMNaey8TWsikaZOwMRwPHQIjcOJM9oV+fi9ZMNw3JsVnbEEbo2gRTDnGEBv8pjyn67hNg==}
dev: true
/@types/lodash/4.14.178:
resolution: {integrity: sha512-0d5Wd09ItQWH1qFbEyQ7oTQ3GZrMfth5JkbN3EvTKLXcHLRDSXeLnlvlOn0wvxVIwK5o2M8JzP/OWz7T3NRsbw==}
dev: true
/@types/mocha/9.0.0:
resolution: {integrity: sha512-scN0hAWyLVAvLR9AyW7HoFF5sJZglyBsbPuHO4fv7JRvfmPBMfp1ozWqOf/e4wwPNxezBZXRfWzMb6iFLgEVRA==}
dev: true
@ -706,6 +714,10 @@ packages:
p-locate: 5.0.0
dev: true
/lodash/4.17.21:
resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==}
dev: false
/log-symbols/4.1.0:
resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==}
engines: {node: '>=10'}

570
src/repo.ts
View File

@ -11,15 +11,14 @@ export interface IDataStore {
}
export type NodeType = "tree" | "blob";
export type NodeHash = string;
export type ObjectID = string;
export type NodeFilename = string;
export type TreeEntry = [NodeType, NodeHash, NodeFilename];
export type TreeEntry = [NodeType, ObjectID, NodeFilename];
export type Commit = {
id: string;
export type ICommit = {
root: string;
before: string;
merge: string;
merge?: string;
date: Date;
};
@ -49,8 +48,57 @@ export const ObjectTypes = {
}
export type ObjectTypeNames = keyof typeof ObjectTypes;
export type TypedObjectData = {
type: ObjectTypeNames,
data: Uint8Array
}
export default class Repository {
//#region static
static sha1(data: Uint8Array) {
const s = new SHA("SHA-1", "UINT8ARRAY");
s.update(data);
return s.getHash("HEX");
}
static sha256(data: Uint8Array) {
const s = new SHA("SHA3-256", "UINT8ARRAY");
s.update(data);
return s.getHash("HEX");
}
static generateObjectID(data: Uint8Array) {
return this.sha1(data);
}
static parseObject(data: Uint8Array | undefined): TypedObjectData | undefined {
if (!data)
return undefined;
let type = new TextDecoder().decode(data.slice(0, 4))
return {
type: type as ObjectTypeNames,
data: data.slice(5)
}
}
static buildObject(data: Uint8Array | string, type: ObjectTypeNames): ObjectTuple {
if (typeof data == "string") {
data = new TextEncoder().encode(data);
}
const objectID = this.generateObjectID(data);
let merged = new Uint8Array(5 + data.length);
merged.set(ObjectTypes[type], 0)
merged.set(data, 5);
return [objectID, merged]
}
//#endregion
//#region local variables
#store: IDataStore;
//#endregion
@ -61,17 +109,7 @@ export default class Repository {
//#region private
private sha1(data: Uint8Array) {
const s = new SHA("SHA-1", "UINT8ARRAY");
s.update(data);
return s.getHash("HEX");
}
private sha256(data: Uint8Array) {
const s = new SHA("SHA3-256", "UINT8ARRAY");
s.update(data);
return s.getHash("HEX");
}
private splitPath(path: string) {
const resolved = Path.resolve(path).slice(1);
@ -105,12 +143,6 @@ export default class Repository {
}
}
private async readTree(id: string): Promise<TreeEntry[]> {
const tree = await this.readObject(id, true);
if (tree == undefined) throw new Error("Invalid treeID");
return this.parseTree(tree);
}
private async makeCommit(treeID: string, old?: Commit, merge?: Commit) {
if (!old) {
// Could be called once more than necessary, if no HEAD exists.
@ -145,7 +177,7 @@ export default class Repository {
if (typeof data == "string") {
data = new TextEncoder().encode(data);
}
const objectID = this.sha1(data);
const objectID = Repository.generateObjectID(data);
let merged = new Uint8Array(5 + data.length);
merged.set(ObjectTypes[type], 0)
@ -165,16 +197,11 @@ export default class Repository {
public async readObjectTyped(id: String) {
let data = await this.#store.get("objects/" + id);
if (!data)
return undefined;
let type = new TextDecoder().decode(data.slice(0, 4))
return {
type: type as ObjectTypeNames,
data: data.slice(5)
}
return Repository.parseObject(data);
}
public async readObject(id: string, string: true): Promise<string | undefined>;
public async readObject(id: string, string?: false): Promise<Uint8Array | undefined>;
public async readObject(id: string, string = false): Promise<Uint8Array | string | undefined> {
@ -204,52 +231,21 @@ export default class Repository {
return type == "comm";
}
parseTree(treeStr: string | Uint8Array): TreeEntry[] {
if (typeof treeStr !== "string")
treeStr = new TextDecoder().decode(treeStr);
return treeStr.split("\n").filter(e => e !== "").map((e) => {
const entry = e.split(" ") as TreeEntry;
const [type] = entry;
switch (type) {
case "blob":
case "tree":
break;
default:
throw new Error("Invalid tree type."); //Might be a newer version or so
}
return entry;
});
private getVirtualRepo() {
return new VirtualRepo(this);
}
parseCommit(id: string, commitStr: string | Uint8Array,) {
if (typeof commitStr !== "string")
commitStr = new TextDecoder().decode(commitStr);
let commit: Commit = { id } as any;
for (const entry of commitStr.split("\n")) {
const [type] = entry.split(" ", 1);
const value = entry.slice(type.length + 1);
switch (type) {
case "tree": // TODO: Simple validity checks
commit.root = value;
break;
case "before": // TODO: Simple validity checks
commit.before = value;
break;
case "merge": // TODO: Simple validity checks
commit.merge = value;
break;
case "date":
commit.date = new Date(value);
break;
}
}
return commit;
async readTree(id: string): Promise<Tree> {
return Tree.load(this.getVirtualRepo(), id);
}
async readBlob(id: string): Promise<Blob> {
return Blob.load(this.getVirtualRepo(), id);
}
async readCommit(id: string): Promise<Commit> {
const commitStr = await this.readObject(id, true);
if (!commitStr)
@ -264,6 +260,13 @@ export default class Repository {
return commit;
}
async getHead() {
if (!(await this.#store.has("HEAD"))) return undefined;
const head = new TextDecoder().decode(await this.#store.get("HEAD"));
return head;
}
async readHead(): Promise<Commit | undefined> {
if (!(await this.#store.has("HEAD"))) return undefined;
const head = new TextDecoder().decode(await this.#store.get("HEAD"));
@ -272,8 +275,36 @@ export default class Repository {
}
async mergeCommits(commit1: string, commit2: string): Promise<string> {
throw new Error("WIP");
// let newCommit = this.makeCommit()
// throw new Error("WIP");
// Find common history (if there). Diff changes.
const getCommitPath = async (commitID: string) => {
let commits: string[] = [];
let com = await this.readCommit(commitID)
if (com.before) {
commits = [...await getCommitPath(com.before), com.before]
}
if (com.merge) { //TODO: Might cause duplicates?
commits = [...await getCommitPath(com.merge), com.merge];
}
return commits;
}
let commits1 = await getCommitPath(commit1);
let commitSet2 = new Set(await getCommitPath(commit2));
let common = commits1.find(commit => commitSet2.has(commit));
if (!common) {
// Complete merge of trees
}
let newTree =
let newCommit = this.makeCommit()
}
@ -288,7 +319,7 @@ export default class Repository {
while (current.before || (till && current.id == till)) {
current = await this.readCommit(current.before);
if (!current)
throw new Error("Repository sems damaged! Can't read commit!");
throw new Error("Repository seems damaged! Can't read commit!");
log.push(current)
}
@ -374,64 +405,73 @@ export default class Repository {
objectID = await this.writeObject(data, "blob");
}
const vr = this.getVirtualRepo()
let blob = data ? Blob.create(vr, data) : undefined;
const lock = await this.#store.getLock();
//TODO: Improve need of locking.
try {
const head = await this.readHead();
const head = await this.getHead();
let com: Commit;
if(head)
com = Commit.load(vr, head);
else
com = Commit.create(vr);
const makeTree = async (treeID: string | undefined, parts: string[]) => {
let tree: TreeEntry[];
if (treeID) {
tree = await this.readTree(treeID);
} else {
tree = [];
}
// const makeTree = async (treeID: string | undefined, parts: string[]) => {
// let tree: TreeEntry[];
// if (treeID) {
// tree = await this.readTree(treeID);
// } else {
// tree = [];
// }
const current = parts[0];
// const current = parts[0];
let existing = tree.findIndex(([, , name]) => name == current);
// let existing = tree.findIndex(([, , name]) => name == current);
let entry: TreeEntry | undefined = undefined;
// let entry: TreeEntry | undefined = undefined;
if (parts.length == 1) {
if (objectID) {
entry = ["blob", objectID, current];
}
} else {
let newTreeID = await makeTree(existing >= 0 ? tree[existing][1] : undefined, parts.slice(1));
if (newTreeID) {
entry = [
"tree",
newTreeID,
current,
];
}
}
// if (parts.length == 1) {
// if (objectID) {
// entry = ["blob", objectID, current];
// }
// } else {
// let newTreeID = await makeTree(existing >= 0 ? tree[existing][1] : undefined, parts.slice(1));
// if (newTreeID) {
// entry = [
// "tree",
// newTreeID,
// current,
// ];
// }
// }
if (existing >= 0) {
let ex = tree[existing];
if (parts.length == 1 && ex[0] == "tree")
throw new Error("This change would overwrite a folder!");
if (entry)
tree[existing] = entry;
else {
tree = [...tree.slice(0, existing), ...tree.slice(existing + 1)];
}
} else {
if (entry)
tree.push(entry);
}
// if (existing >= 0) {
// let ex = tree[existing];
// if (parts.length == 1 && ex[0] == "tree")
// throw new Error("This change would overwrite a folder!");
// if (entry)
// tree[existing] = entry;
// else {
// tree = [...tree.slice(0, existing), ...tree.slice(existing + 1)];
// }
// } else {
// if (entry)
// tree.push(entry);
// }
if (tree.length > 0) {
let treeString = tree.map(([type, hash, name]) => `${type} ${hash} ${name}`).join("\n");
// if (tree.length > 0) {
// let treeString = tree.map(([type, hash, name]) => `${type} ${hash} ${name}`).join("\n");
let newTreeID = await this.writeObject(treeString, "tree");
// let newTreeID = await this.writeObject(treeString, "tree");
return newTreeID;
} else {
return undefined;
}
};
// return newTreeID;
// } else {
// return undefined;
// }
// };
let newTree = await makeTree(head?.root, parts);
if (!newTree) { //TODO: Is this what i want?
@ -448,5 +488,293 @@ export default class Repository {
if (this.#store.close) return this.#store?.close();
}
async applyVirtualCommit() {
//TODO: Maybe this??
}
//#endregion
}
class VirtualRepo {
#virtualObjects = new Map<string, Uint8Array>();
constructor(protected repo: Repository) { }
async getObject(id: string): Promise<TypedObjectData | undefined> {
let local = this.#virtualObjects.get(id);
if (local)
return Repository.parseObject(local);
return this.repo.readObjectTyped(id);
}
async putObject(data: Uint8Array | string, type: ObjectTypeNames): Promise<ObjectID> {
let [id, d] = Repository.buildObject(data, type);
this.#virtualObjects.set(id, d);
return id;
}
async write() {
//TODO: Implement or use applyVirtualCommit of repo
}
}
type ObjectTuple = [string, Uint8Array];
abstract class RepoObject {
public abstract getObject(): Promise<[ObjectID, ObjectID[]] | null>;
public abstract freeze(): void;
}
class Blob extends RepoObject {
#id: ObjectID;
get id(): ObjectID {
return this.#id;
}
constructor(private vr: VirtualRepo, id: string) {
super();
this.#id = id;
}
async getObject() {
this.freeze();
return [this.#id, []] as [ObjectID, ObjectID[]];
}
freeze( ) {
Object.freeze(this);
}
static async load(vr: VirtualRepo, id: string) {
return new Blob(vr, id);
}
static async create(vr: VirtualRepo, data: Uint8Array | string) {
let id = await vr.putObject(data, "blob")
return new Blob(vr, id);
}
}
class Tree extends RepoObject {
trees: Map<string, Tree> = new Map();
blobs: Map<string, Blob> = new Map();
constructor(private vr: VirtualRepo) {
super();
}
static async load(vr: VirtualRepo, id: string) {
const res = await vr.getObject(id);
if (!res)
throw new Error("Invalid TreeID")
if (res.type !== "tree")
throw new Error("This item is not a tree!");
return this.parseTree(vr, res.data);
}
static async create(vr: VirtualRepo) {
return new Tree(vr);
}
setTree(name: string, tree: Tree | null) {
if (this.blobs.has(name))
this.blobs.delete(name); // TODO: Think if error or silent overwrite
if (!tree)
this.trees.delete(name);
else
this.trees.set(name, tree);
}
setBlob(name: string, blob: Blob | null) {
if (this.trees.has(name))
this.trees.delete(name); // TODO: Think if error or silent overwrite
if (!blob)
return this.blobs.delete(name);
else
this.blobs.set(name, blob);
}
freeze() {
Object.freeze(this.blobs)
Object.freeze(this.trees)
Object.freeze(this);
}
async getObject(): Promise<[string, string[]] | null> {
this.freeze();
if (this.trees.size <= 0 && this.blobs.size <= 0)
return null;
let trees = (await Promise.all([...this.trees.entries()].sort(([k1], [k2]) => {
if (k1 < k2)
return -1;
if (k2 > k2)
return 1;
return 0;
}).map(async ([name, element]) => {
let object = await element.getObject();
if (!object)
return undefined;
return {
name,
id: object[0],
dependencies: object[1]
}
}))).filter(e => !!e) as { name: string, id: string, dependencies: string[] }[];
let blobs = (await Promise.all([...this.blobs.entries()].sort(([k1], [k2]) => {
if (k1 < k2)
return -1;
if (k2 > k2)
return 1;
return 0;
}).map(async ([name, element]) => {
let object = await element.getObject();
if (!object)
return undefined;
return {
name,
id: object[0],
dependencies: object[1]
}
}))).filter(e => !!e) as { name: string, id: string, dependencies: string[] }[];
let dependencies = [
...trees.map(e => e?.id),
...trees.map(e => e?.dependencies).flat(1),
...blobs.map(e => e?.id),
...blobs.map(e => e?.dependencies).flat(1),
];
let rows: string[] = [];
trees.forEach((tree) => {
rows.push(`tree ${tree.id} ${tree.name}`);
})
blobs.forEach((blob) => {
rows.push(`blob ${blob.id} ${blob.name}`);
})
let content = rows.join("\n");
let newID = await this.vr.putObject(content, "tree");
//TODO:
// 1. Get Subtrees and their new IDs
// 2. Emit Subtrees/Blobs and new version of itself
return [newID, dependencies];
}
static async parseTree(vr: VirtualRepo, treeStr: string | Uint8Array) {
if (typeof treeStr !== "string")
treeStr = new TextDecoder().decode(treeStr);
let tree = new Tree(vr);
await Promise.all(treeStr.split("\n").filter(e => e !== "").map(async (e) => {
const entry = e.split(" ") as TreeEntry;
const [type, hash, filename] = entry;
switch (type) {
case "blob":
tree.blobs.set(filename, await Blob.load(vr, hash));
break;
case "tree":
tree.trees.set(filename, await Tree.load(vr, hash));
break;
default:
throw new Error("Invalid tree type."); //Might be a newer version or so
}
return entry;
}));
return tree;
}
}
class Commit extends RepoObject {
root?: Tree = undefined;
before?: string = undefined;
date?: Date = undefined;
merge?: string;
constructor(private vr: VirtualRepo) {
super();
}
freeze() {
Object.freeze(this.date);
Object.freeze(this);
}
async getObject() {
if (!this.root)
throw new Error("This commit has no root!");
if (!this.date)
this.date = new Date();
this.freeze()
const rootData = await this.root.getObject();
if (!rootData) //TODO: Might actually happen right now, when the last file is deleted
throw new Error("Invalid tree!")
const [treeID, dependencies] = rootData;
let commitStr =
`tree ${treeID}\ndate ${new Date().toISOString()}\n`;
if (this.before) {
commitStr += `before ${this.before}\n`;
if (this.merge) {
commitStr += `merge ${this.merge}\n`;
}
}
let commitID = await this.vr.putObject(commitStr, "comm");
return [commitID, [...dependencies, treeID]] as [string, string[]];
}
static load(vr: VirtualRepo, id: string) {
let res = await vr.getObject(id);
if (!res)
throw new Error("Invalid ID");
let commitStr = new TextDecoder().decode(res.data);
let commit = new Commit(vr);
for (const entry of commitStr.split("\n")) {
const [type] = entry.split(" ", 1);
const value = entry.slice(type.length + 1);
switch (type) {
case "tree": // TODO: Simple validity checks
commit.root = await Tree.load(vr, value);
break;
case "before": // TODO: Simple validity checks
commit.before = value;
break;
case "merge": // TODO: Simple validity checks
commit.merge = value;
break;
case "date":
commit.date = new Date(value);
break;
}
}
return commit;
}
static create(vr: VirtualRepo) {
return new Commit(vr);
}
}

25
src/repo2.ts Normal file
View File

@ -0,0 +1,25 @@
import * as fs from "fs";
import { compose, multiply, prop, sumBy, map, pipe, pluck, filter, lt } from 'lodash/fp';
class Object {
}
class Blob {
}
class Tree {
}
class Commit {
}
class Repo {
}

51
src/sync.ts
View File

@ -6,7 +6,7 @@ import { ServiceProvider as ServiceProviderClient } from "./com/service_client";
import { ServiceProvider as ServiceProviderServer } from "./com/service_server";
import { PullObjectRequest, PullObjectResponse, PushObjectRequest, PushObjectResponse, Sync as SyncClient, SyncInitRequest, SyncInitResponse } from "./com/sync_client";
import { Sync as SyncServer } from "./com/sync_server";
import Repository, { ObjectTypes } from "./repo";
import Repository, { Commit, ObjectTypes, TreeEntry } from "./repo";
if (typeof btoa === 'undefined') {
global.btoa = function (str) {
@ -189,10 +189,53 @@ export async function startSyncClient(stream: ISimpleStream, repo: Repository) {
let localHead = await repo.readHead()
if (localHead) { // If there is nothing to push, don't push
// Find matching point
let match = undefined;
let toCheck: Commit[] = [localHead];
if (!syncResponse.remoteCommit) { }
let commit: Commit;
while(toCheck.length > 0) {
commit = toCheck.shift() as Commit;
if(commit.id != syncResponse.remoteCommit) { //Not yet at remote commit. Wait
commitsToPush.add(commit.id);
if(commit.before && !commitsToPush.has(commit.before)) // A commit could already be checked by the other part of a merge
toCheck.push(await repo.readCommit(commit.before));
if(commit.merge && !commitsToPush.has(commit.merge))
toCheck.push(await repo.readCommit(commit.merge));
}
}
const traverseTree =async (current: TreeEntry[]) => {
for(const [type, hash, name] of current) {
objectsToPush.add(hash);
if(type =="tree") {
let tree = await repo.readTree(hash);
await traverseTree(tree);
}
}
}
for(const commitId of commitsToPush) {
let commit = await repo.readCommit(commitId);
let rootId = commit.root
objectsToPush.add(rootId);
if(objectsToPush.has(rootId)) {
objectsToPush.add(rootId);
let root = await repo.readTree(rootId);
await traverseTree(root);
}
}
}
for(const objId of objectsToPush) {
let obj = await repo.readObjectTyped(objId)
await service.PushObject({
id: objId,
data: obj?.data as Uint8Array,
type: obj?.type as string
});
}
}