git3.w3q
/
git3-cli
mirror of git3://git3.w3q/git3-cli
Code Activity

fetch done & all mock done

Browse Source
master
cyhhao 2 years ago
parent e3e7e83ac9
commit 8b10f4da49
6 changed files with 147 additions and 103 deletions
Show Stats Download Patch File Download Diff File

2
package.json
Unescape View File

@ -18,6 +18,7 @@
},
"license": "MIT",
"dependencies": {
"buffer-split": "^1.0.0",
"debug": "^4.3.4",
"rxjs": "^6.6.3",
"rxjs-async-map": "^0.2.0",
@ -32,6 +33,7 @@
"clean": "rm -rf ./dist ./bin"
},
"devDependencies": {
"@types/buffer-split": "^1.0.0",
"@types/debug": "^4.1.7",
"@types/node": "^18.11.10",
"ts-node": "^10.9.1",

39
src/git/git-utils.ts
Unescape View File

@ -1,5 +1,6 @@
import childProcess, { spawnSync } from 'child_process'
import * as zlib from "zlib";
import bsplit from 'buffer-split'
import * as zlib from "zlib"
export class GitUtils {
static EMPTY_TREE_HASH: string = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
@ -14,7 +15,7 @@ export class GitUtils {
try {
// const { stdout } = exec(`git ${args.join(" ")}`, { encoding: "utf8" })
const { stdout } = spawnSync("git", args, { encoding: "utf8" })
return stdout
return stdout.trim()
}
catch (e) {
return ""
@ -56,9 +57,22 @@ export class GitUtils {
Buffer.from(size, "utf8"),
Buffer.from("\0"),
contents,
]);
const compressed = zlib.gzipSync(data);
])
const compressed = zlib.gzipSync(data)
return compressed
return data
}
static decodeObject(data: Buffer): string {
const decompressed = zlib.gunzipSync(data)
// const decompressed = data
const splits = bsplit(decompressed, Buffer.from("\0"), true)
const head = bsplit(splits[0], Buffer.from(" "))
const kind = head[0].toString("utf8")
const writeData = Buffer.concat(
splits.slice(1)
)
return this.writeObject(kind, writeData)
}
static isAncestor(ancestor: string, ref: string): boolean {
@ -76,11 +90,11 @@ export class GitUtils {
exclude.push(`^${obj}`)
}
}
const objects = this.commandOutput("rev-list", "--objects", ref, ...exclude);
const objects = this.commandOutput("rev-list", "--objects", ref, ...exclude)
if (!objects) {
return [];
return []
}
return objects.split("\n").map((item) => item.split(" ")[0]).filter(item => item)
return objects.split("\n").map((item) => item.split(/\s/)[0]).filter(item => item)
}
static symbolicRef(ref: string): string {
@ -91,6 +105,7 @@ export class GitUtils {
static writeObject(kind: string, contents: Buffer): string {
let res = spawnSync("git", ["hash-object", "-w", "--stdin", "-t", kind], { input: contents, encoding: "buffer" })
if (res.status != 0) {
console.error(kind, contents)
throw new Error("Failed to write object")
}
else {
@ -111,17 +126,17 @@ export class GitUtils {
let data = this.objectData(sha).toString("utf8").trim()
if (kind == "tag") {
// tag objects reference a single object
let obj = data.split("\n", 1)[0].split(" ")[1]
let obj = data.split("\n", 1)[0].split(/\s/)[1]
return [obj]
}
else if (kind == "commit") {
// commit objects reference a tree and zero or more parents
let lines = data.split("\n")
let tree = lines[0].split(" ")[1]
let tree = lines[0].split(/\s/)[1]
let objs = [tree]
for (let line of lines) {
for (let line of lines.slice(1)) {
if (line.startsWith("parent ")) {
objs.push(line.split(" ")[1])
objs.push(line.split(/\s/)[1])
}
else {
break
@ -138,7 +153,7 @@ export class GitUtils {
let lines = data.split("\n")
// submodules have the mode '160000' and the kind 'commit', we filter them out because
// there is nothing to download and this causes errors
return lines.filter(line => !line.startsWith("160000 commit ")).map(line => line.split(" ")[2])
return lines.filter(line => !line.startsWith("160000 commit ")).map(line => line.split(/\s/)[2])
}
else {
throw new Error(`unexpected git object type: ${kind}`)

129
src/git/git.ts
Unescape View File

@ -1,15 +1,16 @@
import { log } from './log';
import { superpathjoin as join } from 'superpathjoin';
import { ApiBaseParams } from './git-remote-helper';
import { Ref, Status, Storage } from '../storage/storage';
import { GitUtils } from './git-utils';
import { log } from './log'
import { superpathjoin as join } from 'superpathjoin'
import { ApiBaseParams } from './git-remote-helper'
import { Ref, Status, Storage } from '../storage/storage'
import { GitUtils } from './git-utils'
class Git {
gitdir: string
remoteName: string
remoteUrl: string
storage: Storage
refs: Map<string, string> = new Map();
pushed: Map<string, string> = new Map();
refs: Map<string, string> = new Map()
pushed: Map<string, string> = new Map()
head: string | null
constructor(info: ApiBaseParams, storage: Storage) {
this.gitdir = info.gitdir
@ -18,6 +19,7 @@ class Git {
this.storage = storage
this.refs = new Map()
this.pushed = new Map()
this.head = null
}
async do_list(forPush: boolean) {
@ -26,58 +28,29 @@ class Git {
for (let ref of refs) {
if (ref.ref == "HEAD") {
if (!forPush) outLines.push(`@${ref.sha} HEAD\n`)
this.head = ref.sha
}
else {
outLines.push(`${ref.sha} ${ref.ref}\n`)
}
this.refs.set(ref.ref, ref.sha)
}
log("outLines", outLines)
}
return outLines.join("") + "\n"
}
async do_fetch(refs: { ref: string; oid: string }[]) {
async do_fetch(refs: { ref: string, oid: string }[]) {
for (let ref of refs) {
await this.fetch(ref.oid)
}
}
async fetch(oid: string) {
let downloaded = new Set<string>()
let pending = new Set<string>()
let queue = [oid]
while (queue.length > 0 || pending.size > 0) {
if (queue.length > 0) {
let sha = queue.pop() || ""
if (downloaded.has(sha) || pending.has(sha)) continue
if (GitUtils.objectExists(sha)) {
if (sha == GitUtils.EMPTY_TREE_HASH) {
GitUtils.writeObject("tree", Buffer.from(""))
}
if (!GitUtils.historyExists(sha)) {
log("missing part of history from", sha)
queue.push(...GitUtils.referencedObjects(sha))
}
else {
log("already downloaded", sha)
}
}
else {
pending.add(sha)
}
}
else {
}
}
return "\n\n"
}
async do_push(refs: {
src: string;
dst: string;
force: boolean;
src: string
dst: string
force: boolean
}[]): Promise<string> {
let outLines: string[] = []
// let remoteHead = null
@ -100,11 +73,55 @@ class Git {
let symbolicRef = GitUtils.symbolicRef("HEAD")
await this.wirteRef(symbolicRef, "HEAD", true)
}
log("outLines", outLines)
return outLines.join("") + "\n\n"
}
async fetch(oid: string) {
let fetching: Promise<void>[] = []
if (GitUtils.objectExists(oid)) {
if (oid == GitUtils.EMPTY_TREE_HASH) {
GitUtils.writeObject("tree", Buffer.from(""))
}
if (!GitUtils.historyExists(oid)) {
log("missing part of history from", oid)
for (let sha of GitUtils.referencedObjects(oid)) {
fetching.push(this.fetch(sha))
}
}
else {
log("already downloaded", oid)
}
}
else {
let error = await this.download(oid)
if (!error) {
for (let sha of GitUtils.referencedObjects(oid)) {
fetching.push(this.fetch(sha))
}
} else {
fetching.push(this.fetch(oid))
}
}
await Promise.all(fetching)
}
async download(sha: string): Promise<Error | null> {
log("fetching...", sha)
let [status, data] = await this.storage.download(this.objectPath(sha))
if (status == Status.SUCCEED) {
let computedSha = GitUtils.decodeObject(data)
if (computedSha != sha) {
return new Error(`sha mismatch ${computedSha} != ${sha}`)
}
}
else {
return new Error(`download failed ${sha}`)
}
return null
}
async push(src: string, dst: string) {
let force = false
if (src.startsWith("+")) {
@ -114,7 +131,6 @@ class Git {
let present = Array.from(this.refs.values())
present.push(...Array.from(this.pushed.values()))
let objects = GitUtils.listObjects(src, present)
log("listObjects", objects)
for (let obj of objects) {
await this.putObject(obj)
}
@ -139,7 +155,6 @@ class Git {
return "non-fast forward"
}
}
log("setRef", dst, newSha)
let status = await this.storage.setRef(dst, newSha)
if (status == Status.SUCCEED) {
return null
@ -159,23 +174,9 @@ class Git {
}
objectPath(name: string): string {
const prefix = name.slice(0, 2);
const suffix = name.slice(2);
return join("objects", prefix, suffix);
}
async read_symbolic_ref(path: string) {
path = join(this.gitdir, path)
log("fetching symbolic ref: ", path)
try {
const [_, data] = await this.storage.download(path)
let ref = data.toString()
ref = ref.slice("ref: ".length).trim();
return ref;
} catch (e) {
return null;
}
const prefix = name.slice(0, 2)
const suffix = name.slice(2)
return join("objects", prefix, suffix)
}
async get_refs(forPush: boolean): Promise<Ref[]> {

52
src/index.ts
Unescape View File

@ -1,14 +1,14 @@
import GitRemoteHelper from './git/git-remote-helper';
import { ApiBaseParams } from './git/git-remote-helper';
import Git from './git/git';
import { log } from './git/log';
import { ETHStorage } from './storage/ETHStorage';
import GitRemoteHelper from './git/git-remote-helper'
import { ApiBaseParams } from './git/git-remote-helper'
import Git from './git/git'
import { log } from './git/log'
import { ETHStorage } from './storage/ETHStorage'
let git: Git;
let git: Git
GitRemoteHelper({
env: process.env,
stdin: process.stdin,
@ -25,10 +25,10 @@ GitRemoteHelper({
* This needs to return a list of git refs.
*/
list: async (p: {
gitdir: string;
remoteName: string;
remoteUrl: string;
forPush: boolean;
gitdir: string
remoteName: string
remoteUrl: string
forPush: boolean
}) => {
log('list', p)
@ -40,26 +40,28 @@ GitRemoteHelper({
* This should put the requested objects into the `.git`
*/
handleFetch: async (p: {
gitdir: string;
remoteName: string;
remoteUrl: string;
refs: { ref: string; oid: string }[];
gitdir: string
remoteName: string
remoteUrl: string
refs: { ref: string, oid: string }[]
}) => {
log("fetch", p)
return '\n\n';
let out = await git.do_fetch(p.refs)
log("fetch out:\n", out)
return out
},
/**
* This should copy objects from `.git`
*/
handlePush: async (p: {
gitdir: string;
remoteName: string;
remoteUrl: string;
gitdir: string
remoteName: string
remoteUrl: string
refs: {
src: string;
dst: string;
force: boolean;
}[];
src: string
dst: string
force: boolean
}[]
}) => {
log("push", p)
let out = await git.do_push(p.refs)
@ -68,7 +70,7 @@ GitRemoteHelper({
},
},
}).catch((error: any) => {
console.error("wtf");
console.error(error);
console.error("wtf")
console.error(error)
});
})

2
src/storage/ETHStorage.ts
Unescape View File

@ -27,7 +27,6 @@ export class ETHStorage implements Storage {
}
catch (e) {
log("no refs found")
return []
}
@ -53,6 +52,7 @@ export class ETHStorage implements Storage {
let refsJson = await fs.readFile(stPath)
let dict = JSON.parse(refsJson.toString())
delete dict[path]
await fs.writeFile(stPath, JSON.stringify(dict))
return Status.SUCCEED
}

24
yarn.lock
Unescape View File

@ -47,6 +47,13 @@
resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.3.tgz#472eaab5f15c1ffdd7f8628bd4c4f753995ec79e"
integrity sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ==
"@types/buffer-split@^1.0.0":
version "1.0.0"
resolved "https://registry.yarnpkg.com/@types/buffer-split/-/buffer-split-1.0.0.tgz#8c48f2224f95923f42628676358b3d9535de76f8"
integrity sha512-ctMRRk4AX6CnDUUx44hb5i6JMY+aj3cKsGmW3CDwoaVnNoo4E0ZRd+vVeBGjesZkU8cChIIsn77ELFJeTIXIeQ==
dependencies:
"@types/node" "*"
"@types/debug@^4.1.7":
version "4.1.7"
resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.7.tgz#7cc0ea761509124709b8b2d1090d8f6c17aadb82"
@ -59,6 +66,11 @@
resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197"
integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==
"@types/node@*":
version "18.11.13"
resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.13.tgz#dff34f226ec1ac0432ae3b136ec5552bd3b9c0fe"
integrity sha512-IASpMGVcWpUsx5xBOrxMj7Bl8lqfuTY7FKAnPmu5cHkfQVWF8GulWS1jbRqA934qZL35xh5xN/+Xe/i26Bod4w==
"@types/node@^18.11.10":
version "18.11.11"
resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.11.tgz#1d455ac0211549a8409d3cdb371cd55cc971e8dc"
@ -79,6 +91,18 @@ arg@^4.1.0:
resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089"
integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==
buffer-indexof@~0.0.0:
version "0.0.2"
resolved "https://registry.yarnpkg.com/buffer-indexof/-/buffer-indexof-0.0.2.tgz#ed0f36b7ae166a66a7cd174c0467ae8dedf008f5"
integrity sha512-mDMCCNLq1b+ICJ5VzS/D/Ca3bD9/3GxgCp+E+jqLwDTXHxRwQkQDfUOt4pJvmE53+YxfNNtNrxf3k/drNqJKJA==
buffer-split@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/buffer-split/-/buffer-split-1.0.0.tgz#4427dbff53731b61d7a71aba47f503396613784a"
integrity sha512-orCFtxr4KDKi+5AYFDINPvnXtApMCy9moEklNYoGGfXebcFaeKlxK9Wbn6E7HGa3O5hrqYP2ixLXJzxV6RxGXA==
dependencies:
buffer-indexof "~0.0.0"
create-require@^1.1.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333"

Write Preview
Loading…
Cancel
Save