diff --git a/src/git/git-utils.ts b/src/git/git-utils.ts index 7bd3706..79db6e0 100644 --- a/src/git/git-utils.ts +++ b/src/git/git-utils.ts @@ -1,22 +1,19 @@ -import util from 'util' -import childProcess from 'child_process' -const exec = util.promisify(childProcess.exec); +import childProcess, { spawnSync } from 'child_process' import * as zlib from "zlib"; export class GitUtils { - static async commandOK(...args: string[]): Promise { - try { - await exec(`git ${args.join(" ")}`) - return true - } - catch (e) { - return false - } + static EMPTY_TREE_HASH: string = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" + + static commandOK(...args: string[]): boolean { + let res = childProcess.spawnSync("git", args, { encoding: "utf8" }) + return res.status == 0 + } - static async commandOutput(...args: string[]): Promise { + static commandOutput(...args: string[]): string { try { - const { stdout } = await exec(`git ${args.join(" ")}`, { encoding: "utf8" }) + // const { stdout } = exec(`git ${args.join(" ")}`, { encoding: "utf8" }) + const { stdout } = spawnSync("git", args, { encoding: "utf8" }) return stdout } catch (e) { @@ -24,9 +21,9 @@ export class GitUtils { } } - static async commandRaw(...args: string[]): Promise { + static commandRaw(...args: string[]): Buffer { try { - const { stdout } = await exec(`git ${args.join(" ")}`, { encoding: "buffer" }) + const { stdout } = spawnSync("git", args, { encoding: "buffer" }) return stdout } catch (e) { @@ -34,25 +31,25 @@ export class GitUtils { } } - static async objectExists(sha: string): Promise { - return await this.commandOK("cat-file", "-e", sha) + static objectExists(sha: string): boolean { + return this.commandOK("cat-file", "-e", sha) } - static async objectKind(sha: string): Promise { - return await this.commandOutput("cat-file", "-t", sha) + static objectKind(sha: string): string { + return this.commandOutput("cat-file", "-t", sha) } - static async objectData(sha: string, kind: string): Promise { + static objectData(sha: string, kind: string | null = null): Buffer { if (kind) { - return await this.commandRaw("cat-file", kind, sha) + return this.commandRaw("cat-file", kind, sha) } else { - return await this.commandRaw("cat-file", "-p", sha) + return this.commandRaw("cat-file", "-p", sha) } } - static async encodeObject(sha: string): Promise { - let kind = await this.objectKind(sha) - let size = await this.commandOutput("cat-file", "-s", sha) - let contents = await this.objectData(sha, kind) + static encodeObject(sha: string): Buffer { + let kind = this.objectKind(sha) + let size = this.commandOutput("cat-file", "-s", sha) + let contents = this.objectData(sha, kind) const data = Buffer.concat([ Buffer.from(kind, "utf8"), Buffer.from(" "), @@ -64,30 +61,87 @@ export class GitUtils { return compressed } - static async isAncestor(ancestor: string, ref: string): Promise { - return await this.commandOK("merge-base", "--is-ancestor", ancestor, ref) + static isAncestor(ancestor: string, ref: string): boolean { + return this.commandOK("merge-base", "--is-ancestor", ancestor, ref) } - static async refValue(ref: string): Promise { - let sha = await this.commandOutput("rev-parse", ref) + static refValue(ref: string): string { + let sha = this.commandOutput("rev-parse", ref) return sha.trim() } - static async listObjects(ref: string, excludeList: string[]): Promise { + static listObjects(ref: string, excludeList: string[]): string[] { let exclude: string[] = [] for (let obj of excludeList) { - if (!await this.objectExists(obj)) { + if (this.objectExists(obj)) { exclude.push(`^${obj}`) } } - const objects = await this.commandOutput("rev-list", "--objects", ref, ...exclude); + const objects = this.commandOutput("rev-list", "--objects", ref, ...exclude); if (!objects) { return []; } return objects.split("\n").map((item) => item.split(" ")[0]).filter(item => item) } - static async symbolicRef(ref: string): Promise { - let path = await this.commandOutput("symbolic-ref", ref) + static symbolicRef(ref: string): string { + let path = this.commandOutput("symbolic-ref", ref) return path.trim() } + + static writeObject(kind: string, contents: Buffer): string { + let res = spawnSync("git", ["hash-object", "-w", "--stdin", "-t", kind], { input: contents, encoding: "buffer" }) + if (res.status != 0) { + throw new Error("Failed to write object") + } + else { + return res.stdout.toString("utf8").trim() + } + } + + static historyExists(sha: string): boolean { + return this.commandOK("rev-list", "--objects", sha) + } + + static referencedObjects(sha: string): string[] { + let kind = this.objectKind(sha) + if (kind == "blob") { + // blob objects do not reference any other objects + return [] + } + let data = this.objectData(sha).toString("utf8").trim() + if (kind == "tag") { + // tag objects reference a single object + let obj = data.split("\n", 1)[0].split(" ")[1] + return [obj] + } + else if (kind == "commit") { + // commit objects reference a tree and zero or more parents + let lines = data.split("\n") + let tree = lines[0].split(" ")[1] + let objs = [tree] + for (let line of lines) { + if (line.startsWith("parent ")) { + objs.push(line.split(" ")[1]) + } + else { + break + } + } + return objs + } + else if (kind == "tree") { + // tree objects reference zero or more trees and blobs, or submodules + if (!data) { + // empty tree + return [] + } + let lines = data.split("\n") + // submodules have the mode '160000' and the kind 'commit', we filter them out because + // there is nothing to download and this causes errors + return lines.filter(line => !line.startsWith("160000 commit ")).map(line => line.split(" ")[2]) + } + else { + throw new Error(`unexpected git object type: ${kind}`) + } + } } \ No newline at end of file diff --git a/src/git/git.ts b/src/git/git.ts index 82d3142..796cb82 100644 --- a/src/git/git.ts +++ b/src/git/git.ts @@ -9,6 +9,7 @@ class Git { remoteUrl: string storage: Storage refs: Map = new Map(); + pushed: Map = new Map(); constructor(info: ApiBaseParams, storage: Storage) { this.gitdir = info.gitdir @@ -16,6 +17,7 @@ class Git { this.remoteUrl = info.remoteUrl this.storage = storage this.refs = new Map() + this.pushed = new Map() } async do_list(forPush: boolean) { @@ -36,7 +38,40 @@ class Git { } async do_fetch(refs: { ref: string; oid: string }[]) { + for (let ref of refs) { + await this.fetch(ref.oid) + } + } + + async fetch(oid: string) { + let downloaded = new Set() + let pending = new Set() + let queue = [oid] + + while (queue.length > 0 || pending.size > 0) { + if (queue.length > 0) { + let sha = queue.pop() || "" + if (downloaded.has(sha) || pending.has(sha)) continue + if (GitUtils.objectExists(sha)) { + if (sha == GitUtils.EMPTY_TREE_HASH) { + GitUtils.writeObject("tree", Buffer.from("")) + } + if (!GitUtils.historyExists(sha)) { + log("missing part of history from", sha) + queue.push(...GitUtils.referencedObjects(sha)) + } + else { + log("already downloaded", sha) + } + } + else { + pending.add(sha) + } + } + else { + } + } } async do_push(refs: { @@ -48,14 +83,21 @@ class Git { // let remoteHead = null for (let ref of refs) { if (ref.src == "") { - this.storage.delete(ref.dst) + if (this.refs.get("HEAD") == ref.dst) { + return `error ${ref.dst} refusing to delete the current branch: ${ref.dst}` + "\n\n" + } + log("deleting ref", ref.dst) + this.storage.removeRef(ref.dst) + + this.refs.delete(ref.dst) + this.pushed.delete(ref.dst) } else { outLines.push(await this.push(ref.src, ref.dst) + "\n") } } if (this.refs.size == 0) { // first push - let symbolicRef = await GitUtils.symbolicRef("HEAD") + let symbolicRef = GitUtils.symbolicRef("HEAD") await this.wirteRef(symbolicRef, "HEAD", true) } log("outLines", outLines) @@ -70,14 +112,16 @@ class Git { force = true } let present = Array.from(this.refs.values()) - let objects = await GitUtils.listObjects(src, present) + present.push(...Array.from(this.pushed.values())) + let objects = GitUtils.listObjects(src, present) log("listObjects", objects) for (let obj of objects) { await this.putObject(obj) } - let sha = await GitUtils.refValue(src) + let sha = GitUtils.refValue(src) let err = await this.wirteRef(sha, dst, force) if (!err) { + this.pushed.set(dst, sha) return `ok ${dst}` } else { return `error ${dst} ${err}` @@ -85,10 +129,9 @@ class Git { } async wirteRef(newSha: string, dst: string, force: boolean): Promise { - let sha = this.refs.get(dst) if (sha) { - if (!await GitUtils.objectExists(sha)) { + if (!GitUtils.objectExists(sha)) { return "fetch first" } let isFastForward = GitUtils.isAncestor(sha, newSha) @@ -108,7 +151,7 @@ class Git { } async putObject(sha: string) { - let data = await GitUtils.encodeObject(sha) + let data = GitUtils.encodeObject(sha) let path = this.objectPath(sha) log("writing...", path, sha) let status = await this.storage.upload(path, data) @@ -120,6 +163,7 @@ class Git { const suffix = name.slice(2); return join("objects", prefix, suffix); } + async read_symbolic_ref(path: string) { path = join(this.gitdir, path) log("fetching symbolic ref: ", path) diff --git a/src/index.ts b/src/index.ts index 12f8112..f5ca454 100644 --- a/src/index.ts +++ b/src/index.ts @@ -30,7 +30,7 @@ GitRemoteHelper({ remoteUrl: string; forPush: boolean; }) => { - log('list log', p) + log('list', p) let out = await git.do_list(p.forPush) log("list out:\n", out) diff --git a/src/storage/ETHStorage.ts b/src/storage/ETHStorage.ts index 8420e4e..6f930a9 100644 --- a/src/storage/ETHStorage.ts +++ b/src/storage/ETHStorage.ts @@ -48,7 +48,7 @@ export class ETHStorage implements Storage { await fs.writeFile(stPath, JSON.stringify(dict)) return Status.SUCCEED } - async delRef(path: string): Promise { + async removeRef(path: string): Promise { let stPath = join(mockPath, "refs.json") let refsJson = await fs.readFile(stPath) let dict = JSON.parse(refsJson.toString()) @@ -56,7 +56,7 @@ export class ETHStorage implements Storage { return Status.SUCCEED } - async delete(path: string): Promise { + async remove(path: string): Promise { throw new Error("Method not implemented."); } async download(path: string): Promise<[Status, Buffer]> { diff --git a/src/storage/storage.ts b/src/storage/storage.ts index 137f1f0..35d0f8f 100644 --- a/src/storage/storage.ts +++ b/src/storage/storage.ts @@ -16,9 +16,9 @@ export interface Storage { download(path: string): Promise<[Status, Buffer]> upload(path: string, file: Buffer): Promise - delete(path: string): Promise + remove(path: string): Promise listRefs(): Promise setRef(path: string, sha: string): Promise - delRef(path: string): Promise + removeRef(path: string): Promise } \ No newline at end of file