opt slow download

package.json

@@ -1,6 +1,6 @@
 {
     "name": "git3-cli",
-    "version": "0.1.1",
+    "version": "0.1.3",
     "license": "MIT",
     "type": "module",
     "engines": {

src/config/ipfs.ts

@@ -1,7 +1,9 @@
 export default {
     gateways: [
-        "https://ipfs.io/ipfs/",
-        "https://cloudflare-ipfs.com/ipfs/",
-        "https://gateway.ipfs.io/ipfs/",
+        "https://{cid}.ipfs.nftstorage.link/",
+        "https://{cid}.ipfs.w3s.link",
+        "https://ipfs.io/ipfs/{cid}",
+        "https://cloudflare-ipfs.com/ipfs/{cid}",
+        "https://fleek.ipfs.io/ipfs/{cid}",
     ],
 }

src/git-remote-git3/git.ts

@@ -11,6 +11,7 @@ class Git {
     refs: Map<string, string> = new Map()
     pushed: Map<string, string> = new Map()
     head: string | null
+    fetchPending: Map<string, number> = new Map()
 
     constructor(info: ApiBaseParams, storage: Storage) {
         this.gitdir = info.gitdir
@@ -92,6 +93,13 @@ class Git {
     }
 
     async fetch(oid: string) {
+        if (this.fetchPending.has(oid)) {
+            let cnt = this.fetchPending.get(oid)!
+            this.fetchPending.set(oid, cnt + 1)
+            return
+        }
+        this.fetchPending.set(oid, 1)
+
         let fetching: Promise<void>[] = []
         if (GitUtils.objectExists(oid)) {
             if (oid == GitUtils.EMPTY_TREE_HASH) {
@@ -112,10 +120,12 @@ class Git {
                     fetching.push(this.fetch(sha))
                 }
             } else {
-                fetching.push(this.fetch(oid))
+                throw error
+                //fetching.push(this.fetch(oid))
             }
         }
         await Promise.all(fetching)
+        this.fetchPending.set(oid, 0)
     }
 
     async download(sha: string): Promise<Error | null> {

src/storage/SLIStorage.ts

@@ -5,7 +5,7 @@ import ipfsConf from "../config/ipfs.js"
 import axios from "axios"
 import { Git3Protocol } from "../common/git3-protocol.js"
 import { QueueTask, Retrier } from "../common/queue-task.js"
-
+import https from "https"
 export class SLIStorage implements Storage {
     repoName: string
     wallet: ethers.Wallet
@@ -65,13 +65,15 @@ export class SLIStorage implements Storage {
         const buffer = Buffer.from(res[0].slice(2), "hex")
         const cid = buffer.toString("utf8")
         for (let i = 0; i < ipfsConf.gateways.length; i++) {
-            let gateway = ipfsConf.gateways[Math.floor(Math.random() * ipfsConf.gateways.length)] //random get rpc
+            let gateway = ipfsConf.gateways[i] // don't random, try every gateway
             try {
                 let resault = await Retrier(
                     async () => {
-                        const TIMEOUT = 15
-                        let response = await axios.get(gateway + cid, {
+                        const TIMEOUT = 3
+                        const CONNECT_TIMEOUT = 3
+                        let response = await axios.get(gateway.replace("{cid}", cid), {
                             responseType: "arraybuffer",
+                            httpsAgent: new https.Agent({ timeout: CONNECT_TIMEOUT * 1000 }),
                             timeout: TIMEOUT * 1000,
                         })
                         if (response.status === 200) {
@@ -83,8 +85,9 @@ export class SLIStorage implements Storage {
                     { maxRetry: 3 }
                 )
                 return [Status.SUCCEED, resault]
-            } catch (e) {
+            } catch (e: any) {
                 //pass
+                // console.error("download err:", e.message, gateway.replace("{cid}", cid))
             }
         }