From 8b10f4da49106de95593ac5d6374a49e2670e365 Mon Sep 17 00:00:00 2001 From: cyhhao Date: Mon, 12 Dec 2022 17:27:07 +0800 Subject: [PATCH] fetch done & all mock done --- package.json | 4 +- src/git/git-utils.ts | 39 ++++++++---- src/git/git.ts | 129 +++++++++++++++++++------------------- src/index.ts | 52 +++++++-------- src/storage/ETHStorage.ts | 2 +- yarn.lock | 24 +++++++ 6 files changed, 147 insertions(+), 103 deletions(-) diff --git a/package.json b/package.json index 4eb2548..09fcf71 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ }, "license": "MIT", "dependencies": { + "buffer-split": "^1.0.0", "debug": "^4.3.4", "rxjs": "^6.6.3", "rxjs-async-map": "^0.2.0", @@ -32,9 +33,10 @@ "clean": "rm -rf ./dist ./bin" }, "devDependencies": { + "@types/buffer-split": "^1.0.0", "@types/debug": "^4.1.7", "@types/node": "^18.11.10", "ts-node": "^10.9.1", "typescript": "^4.9.3" } -} \ No newline at end of file +} diff --git a/src/git/git-utils.ts b/src/git/git-utils.ts index 79db6e0..8fe3b88 100644 --- a/src/git/git-utils.ts +++ b/src/git/git-utils.ts @@ -1,5 +1,6 @@ import childProcess, { spawnSync } from 'child_process' -import * as zlib from "zlib"; +import bsplit from 'buffer-split' +import * as zlib from "zlib" export class GitUtils { static EMPTY_TREE_HASH: string = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" @@ -14,7 +15,7 @@ export class GitUtils { try { // const { stdout } = exec(`git ${args.join(" ")}`, { encoding: "utf8" }) const { stdout } = spawnSync("git", args, { encoding: "utf8" }) - return stdout + return stdout.trim() } catch (e) { return "" @@ -56,9 +57,22 @@ export class GitUtils { Buffer.from(size, "utf8"), Buffer.from("\0"), contents, - ]); - const compressed = zlib.gzipSync(data); + ]) + const compressed = zlib.gzipSync(data) return compressed + return data + } + + static decodeObject(data: Buffer): string { + const decompressed = zlib.gunzipSync(data) + // const decompressed = data + const splits = bsplit(decompressed, Buffer.from("\0"), true) + const head = bsplit(splits[0], Buffer.from(" ")) + const kind = head[0].toString("utf8") + const writeData = Buffer.concat( + splits.slice(1) + ) + return this.writeObject(kind, writeData) } static isAncestor(ancestor: string, ref: string): boolean { @@ -76,11 +90,11 @@ export class GitUtils { exclude.push(`^${obj}`) } } - const objects = this.commandOutput("rev-list", "--objects", ref, ...exclude); + const objects = this.commandOutput("rev-list", "--objects", ref, ...exclude) if (!objects) { - return []; + return [] } - return objects.split("\n").map((item) => item.split(" ")[0]).filter(item => item) + return objects.split("\n").map((item) => item.split(/\s/)[0]).filter(item => item) } static symbolicRef(ref: string): string { @@ -91,6 +105,7 @@ export class GitUtils { static writeObject(kind: string, contents: Buffer): string { let res = spawnSync("git", ["hash-object", "-w", "--stdin", "-t", kind], { input: contents, encoding: "buffer" }) if (res.status != 0) { + console.error(kind, contents) throw new Error("Failed to write object") } else { @@ -111,17 +126,17 @@ export class GitUtils { let data = this.objectData(sha).toString("utf8").trim() if (kind == "tag") { // tag objects reference a single object - let obj = data.split("\n", 1)[0].split(" ")[1] + let obj = data.split("\n", 1)[0].split(/\s/)[1] return [obj] } else if (kind == "commit") { // commit objects reference a tree and zero or more parents let lines = data.split("\n") - let tree = lines[0].split(" ")[1] + let tree = lines[0].split(/\s/)[1] let objs = [tree] - for (let line of lines) { + for (let line of lines.slice(1)) { if (line.startsWith("parent ")) { - objs.push(line.split(" ")[1]) + objs.push(line.split(/\s/)[1]) } else { break @@ -138,7 +153,7 @@ export class GitUtils { let lines = data.split("\n") // submodules have the mode '160000' and the kind 'commit', we filter them out because // there is nothing to download and this causes errors - return lines.filter(line => !line.startsWith("160000 commit ")).map(line => line.split(" ")[2]) + return lines.filter(line => !line.startsWith("160000 commit ")).map(line => line.split(/\s/)[2]) } else { throw new Error(`unexpected git object type: ${kind}`) diff --git a/src/git/git.ts b/src/git/git.ts index 796cb82..735dc0c 100644 --- a/src/git/git.ts +++ b/src/git/git.ts @@ -1,15 +1,16 @@ -import { log } from './log'; -import { superpathjoin as join } from 'superpathjoin'; -import { ApiBaseParams } from './git-remote-helper'; -import { Ref, Status, Storage } from '../storage/storage'; -import { GitUtils } from './git-utils'; +import { log } from './log' +import { superpathjoin as join } from 'superpathjoin' +import { ApiBaseParams } from './git-remote-helper' +import { Ref, Status, Storage } from '../storage/storage' +import { GitUtils } from './git-utils' class Git { gitdir: string remoteName: string remoteUrl: string storage: Storage - refs: Map = new Map(); - pushed: Map = new Map(); + refs: Map = new Map() + pushed: Map = new Map() + head: string | null constructor(info: ApiBaseParams, storage: Storage) { this.gitdir = info.gitdir @@ -18,6 +19,7 @@ class Git { this.storage = storage this.refs = new Map() this.pushed = new Map() + this.head = null } async do_list(forPush: boolean) { @@ -26,58 +28,29 @@ class Git { for (let ref of refs) { if (ref.ref == "HEAD") { if (!forPush) outLines.push(`@${ref.sha} HEAD\n`) + this.head = ref.sha } else { outLines.push(`${ref.sha} ${ref.ref}\n`) + this.refs.set(ref.ref, ref.sha) } - this.refs.set(ref.ref, ref.sha) + } - log("outLines", outLines) return outLines.join("") + "\n" } - async do_fetch(refs: { ref: string; oid: string }[]) { + async do_fetch(refs: { ref: string, oid: string }[]) { for (let ref of refs) { await this.fetch(ref.oid) } - } - - async fetch(oid: string) { - let downloaded = new Set() - let pending = new Set() - let queue = [oid] - - while (queue.length > 0 || pending.size > 0) { - if (queue.length > 0) { - let sha = queue.pop() || "" - if (downloaded.has(sha) || pending.has(sha)) continue - if (GitUtils.objectExists(sha)) { - if (sha == GitUtils.EMPTY_TREE_HASH) { - GitUtils.writeObject("tree", Buffer.from("")) - } - if (!GitUtils.historyExists(sha)) { - log("missing part of history from", sha) - queue.push(...GitUtils.referencedObjects(sha)) - } - else { - log("already downloaded", sha) - } - } - else { - pending.add(sha) - } - } - else { - - } - } + return "\n\n" } async do_push(refs: { - src: string; - dst: string; - force: boolean; + src: string + dst: string + force: boolean }[]): Promise { let outLines: string[] = [] // let remoteHead = null @@ -100,11 +73,55 @@ class Git { let symbolicRef = GitUtils.symbolicRef("HEAD") await this.wirteRef(symbolicRef, "HEAD", true) } - log("outLines", outLines) return outLines.join("") + "\n\n" } + async fetch(oid: string) { + let fetching: Promise[] = [] + if (GitUtils.objectExists(oid)) { + if (oid == GitUtils.EMPTY_TREE_HASH) { + GitUtils.writeObject("tree", Buffer.from("")) + } + if (!GitUtils.historyExists(oid)) { + log("missing part of history from", oid) + for (let sha of GitUtils.referencedObjects(oid)) { + fetching.push(this.fetch(sha)) + } + } + else { + log("already downloaded", oid) + } + } + else { + let error = await this.download(oid) + if (!error) { + for (let sha of GitUtils.referencedObjects(oid)) { + fetching.push(this.fetch(sha)) + } + } else { + fetching.push(this.fetch(oid)) + } + + } + await Promise.all(fetching) + } + + async download(sha: string): Promise { + log("fetching...", sha) + let [status, data] = await this.storage.download(this.objectPath(sha)) + if (status == Status.SUCCEED) { + let computedSha = GitUtils.decodeObject(data) + if (computedSha != sha) { + return new Error(`sha mismatch ${computedSha} != ${sha}`) + } + } + else { + return new Error(`download failed ${sha}`) + } + return null + } + async push(src: string, dst: string) { let force = false if (src.startsWith("+")) { @@ -114,7 +131,6 @@ class Git { let present = Array.from(this.refs.values()) present.push(...Array.from(this.pushed.values())) let objects = GitUtils.listObjects(src, present) - log("listObjects", objects) for (let obj of objects) { await this.putObject(obj) } @@ -139,7 +155,6 @@ class Git { return "non-fast forward" } } - log("setRef", dst, newSha) let status = await this.storage.setRef(dst, newSha) if (status == Status.SUCCEED) { return null @@ -159,23 +174,9 @@ class Git { } objectPath(name: string): string { - const prefix = name.slice(0, 2); - const suffix = name.slice(2); - return join("objects", prefix, suffix); - } - - async read_symbolic_ref(path: string) { - path = join(this.gitdir, path) - log("fetching symbolic ref: ", path) - - try { - const [_, data] = await this.storage.download(path) - let ref = data.toString() - ref = ref.slice("ref: ".length).trim(); - return ref; - } catch (e) { - return null; - } + const prefix = name.slice(0, 2) + const suffix = name.slice(2) + return join("objects", prefix, suffix) } async get_refs(forPush: boolean): Promise { diff --git a/src/index.ts b/src/index.ts index f5ca454..07fafa8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,14 +1,14 @@ -import GitRemoteHelper from './git/git-remote-helper'; -import { ApiBaseParams } from './git/git-remote-helper'; -import Git from './git/git'; -import { log } from './git/log'; -import { ETHStorage } from './storage/ETHStorage'; +import GitRemoteHelper from './git/git-remote-helper' +import { ApiBaseParams } from './git/git-remote-helper' +import Git from './git/git' +import { log } from './git/log' +import { ETHStorage } from './storage/ETHStorage' -let git: Git; +let git: Git GitRemoteHelper({ env: process.env, stdin: process.stdin, @@ -25,10 +25,10 @@ GitRemoteHelper({ * This needs to return a list of git refs. */ list: async (p: { - gitdir: string; - remoteName: string; - remoteUrl: string; - forPush: boolean; + gitdir: string + remoteName: string + remoteUrl: string + forPush: boolean }) => { log('list', p) @@ -40,26 +40,28 @@ GitRemoteHelper({ * This should put the requested objects into the `.git` */ handleFetch: async (p: { - gitdir: string; - remoteName: string; - remoteUrl: string; - refs: { ref: string; oid: string }[]; + gitdir: string + remoteName: string + remoteUrl: string + refs: { ref: string, oid: string }[] }) => { log("fetch", p) - return '\n\n'; + let out = await git.do_fetch(p.refs) + log("fetch out:\n", out) + return out }, /** * This should copy objects from `.git` */ handlePush: async (p: { - gitdir: string; - remoteName: string; - remoteUrl: string; + gitdir: string + remoteName: string + remoteUrl: string refs: { - src: string; - dst: string; - force: boolean; - }[]; + src: string + dst: string + force: boolean + }[] }) => { log("push", p) let out = await git.do_push(p.refs) @@ -68,7 +70,7 @@ GitRemoteHelper({ }, }, }).catch((error: any) => { - console.error("wtf"); - console.error(error); + console.error("wtf") + console.error(error) -}); \ No newline at end of file +}) \ No newline at end of file diff --git a/src/storage/ETHStorage.ts b/src/storage/ETHStorage.ts index 6f930a9..75b37bb 100644 --- a/src/storage/ETHStorage.ts +++ b/src/storage/ETHStorage.ts @@ -27,7 +27,6 @@ export class ETHStorage implements Storage { } catch (e) { - log("no refs found") return [] } @@ -53,6 +52,7 @@ export class ETHStorage implements Storage { let refsJson = await fs.readFile(stPath) let dict = JSON.parse(refsJson.toString()) delete dict[path] + await fs.writeFile(stPath, JSON.stringify(dict)) return Status.SUCCEED } diff --git a/yarn.lock b/yarn.lock index b9f9eb9..f9020ff 100644 --- a/yarn.lock +++ b/yarn.lock @@ -47,6 +47,13 @@ resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.3.tgz#472eaab5f15c1ffdd7f8628bd4c4f753995ec79e" integrity sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ== +"@types/buffer-split@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@types/buffer-split/-/buffer-split-1.0.0.tgz#8c48f2224f95923f42628676358b3d9535de76f8" + integrity sha512-ctMRRk4AX6CnDUUx44hb5i6JMY+aj3cKsGmW3CDwoaVnNoo4E0ZRd+vVeBGjesZkU8cChIIsn77ELFJeTIXIeQ== + dependencies: + "@types/node" "*" + "@types/debug@^4.1.7": version "4.1.7" resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.7.tgz#7cc0ea761509124709b8b2d1090d8f6c17aadb82" @@ -59,6 +66,11 @@ resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197" integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA== +"@types/node@*": + version "18.11.13" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.13.tgz#dff34f226ec1ac0432ae3b136ec5552bd3b9c0fe" + integrity sha512-IASpMGVcWpUsx5xBOrxMj7Bl8lqfuTY7FKAnPmu5cHkfQVWF8GulWS1jbRqA934qZL35xh5xN/+Xe/i26Bod4w== + "@types/node@^18.11.10": version "18.11.11" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.11.tgz#1d455ac0211549a8409d3cdb371cd55cc971e8dc" @@ -79,6 +91,18 @@ arg@^4.1.0: resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089" integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA== +buffer-indexof@~0.0.0: + version "0.0.2" + resolved "https://registry.yarnpkg.com/buffer-indexof/-/buffer-indexof-0.0.2.tgz#ed0f36b7ae166a66a7cd174c0467ae8dedf008f5" + integrity sha512-mDMCCNLq1b+ICJ5VzS/D/Ca3bD9/3GxgCp+E+jqLwDTXHxRwQkQDfUOt4pJvmE53+YxfNNtNrxf3k/drNqJKJA== + +buffer-split@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/buffer-split/-/buffer-split-1.0.0.tgz#4427dbff53731b61d7a71aba47f503396613784a" + integrity sha512-orCFtxr4KDKi+5AYFDINPvnXtApMCy9moEklNYoGGfXebcFaeKlxK9Wbn6E7HGa3O5hrqYP2ixLXJzxV6RxGXA== + dependencies: + buffer-indexof "~0.0.0" + create-require@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333"