diff --git a/src/common/queue-task.ts b/src/common/queue-task.ts index 5900432..2d29bcc 100644 --- a/src/common/queue-task.ts +++ b/src/common/queue-task.ts @@ -1,3 +1,10 @@ +export interface QueueTaskOptions { + maxRetry?: number + queueInterval?: number + maxPending?: number + retryInterval?: number +} + export class QueueTask { maxRetry: number = 0 queueInterval: number = 0 @@ -11,7 +18,12 @@ export class QueueTask { checkPointResolve: any index: number = 0 - constructor({ maxRetry = 10, queueInterval = 0, maxPending = 0, retryInterval = 0 }) { + constructor({ + maxRetry = 10, + queueInterval = 0, + maxPending = 0, + retryInterval = 0, + }: QueueTaskOptions) { this.maxRetry = maxRetry this.queueInterval = queueInterval this.maxPending = maxPending @@ -41,7 +53,8 @@ export class QueueTask { setTimeout(() => { resolve(true) }, delta) - this.lastTaskScheduled = delta + // console.error("wait delta", delta, "ms") + this.lastTaskScheduled = now + delta return } else { this.lastTaskScheduled = now @@ -55,14 +68,17 @@ export class QueueTask { let lastError for (let i = 0; i < retry; i++) { this.index++ - let start = Date.now().valueOf() - console.error("wait-" + this.index) + // let start = Date.now().valueOf() + // let index = this.index + // console.error("wait-" + index) await this.tickThread() - console.log("run-" + this.index, Date.now().valueOf() - start, "ms") + // console.error("run-" + index, Date.now().valueOf() - start, "ms") this.pending++ try { + // let start = Date.now().valueOf() let res = await func() + // console.error("done-" + index, Date.now().valueOf() - start, "ms") this.pending-- this.checkPointResolve(true) return res @@ -76,3 +92,10 @@ export class QueueTask { if (lastError) throw lastError } } + +export function Retrier( + func: Function, + { maxRetry = 10, queueInterval = 0, maxPending = 0, retryInterval = 0 }: QueueTaskOptions +): Promise { + return new QueueTask({ maxRetry, queueInterval, maxPending, retryInterval }).run(func) +} diff --git a/src/common/tx-manager.ts b/src/common/tx-manager.ts index 8dc981b..64f4389 100644 --- a/src/common/tx-manager.ts +++ b/src/common/tx-manager.ts @@ -1,4 +1,5 @@ import { ethers } from "ethers" +import { Retrier } from "./queue-task.js" export class TxManager { contract: ethers.Contract @@ -88,14 +89,19 @@ export class TxManager { async SendCall(_method: string, _args: any[]): Promise { let lastError: any = null - const nonce = await this.getNonce() + const nonce = await Retrier(this.getNonce.bind(this), { maxRetry: this.boardcastTimes }) if (this.queueCurrNonce < 0) this.queueCurrNonce = nonce let unsignedTx = await this.contract.populateTransaction[_method](..._args) unsignedTx.nonce = nonce unsignedTx.chainId = this.chainId // estimateGas check - let gasLimit = await this.contract.provider.estimateGas(unsignedTx) + let gasLimit = await Retrier( + async () => await this.contract.provider.estimateGas(unsignedTx), + { + maxRetry: this.boardcastTimes, + } + ) unsignedTx.gasLimit = gasLimit.mul((this.gasLimitRatio * 100) | 0).div(100) let retryRBF = this.rbfTimes let rbfCount = 0 @@ -106,7 +112,9 @@ export class TxManager { // set gas price let price try { - price = await this.FreshBaseGas() + price = await await Retrier(this.FreshBaseGas.bind(this), { + maxRetry: this.boardcastTimes, + }) } catch (e) { price = this.price } finally { diff --git a/src/config/evm-network.ts b/src/config/evm-network.ts index 069d669..1f061d7 100644 --- a/src/config/evm-network.ts +++ b/src/config/evm-network.ts @@ -1,52 +1,52 @@ // from https://chainid.network/chains.json const evmNetworks: Record = { - 1: { - name: "Ethereum", - nativeCurrency: { - name: "Ether", - symbol: "ETH", - decimals: 18, - }, - rpc: [ - "https://rpc.flashbots.net", - "https://singapore.rpc.blxrbdn.com", - "https://rpc.ankr.com/eth", - ], - explorers: [ - { - name: "etherscan", - url: "https://etherscan.io", - standard: "EIP3091", - }, - ], - txConst: { - blockTimeSec: 15, - }, - contracts: { git3: "" }, - }, - 5: { - name: "Goerli", - rpc: ["https://eth-goerli.g.alchemy.com/v2/asrXwNuiK9my-cZJYZ_ooo4q-lDw8HLm"], - nativeCurrency: { - name: "Goerli Ether", - symbol: "ETH", - decimals: 18, - }, - explorers: [ - { - name: "etherscan-goerli", - url: "https://goerli.etherscan.io", - standard: "EIP3091", - }, - ], - txConst: { - blockTimeSec: 12, - rbfTimes: 6, - boardcastTimes: 5, - }, - contracts: { git3: "0x51bb7F23193b88696D25EAec7E3293a2C96e55Ee" }, - }, + // 1: { + // name: "Ethereum", + // nativeCurrency: { + // name: "Ether", + // symbol: "ETH", + // decimals: 18, + // }, + // rpc: [ + // "https://rpc.flashbots.net", + // "https://singapore.rpc.blxrbdn.com", + // "https://rpc.ankr.com/eth", + // ], + // explorers: [ + // { + // name: "etherscan", + // url: "https://etherscan.io", + // standard: "EIP3091", + // }, + // ], + // txConst: { + // blockTimeSec: 15, + // }, + // contracts: { git3: "" }, + // }, + // 5: { + // name: "Goerli", + // rpc: ["https://eth-goerli.g.alchemy.com/v2/asrXwNuiK9my-cZJYZ_ooo4q-lDw8HLm"], + // nativeCurrency: { + // name: "Goerli Ether", + // symbol: "ETH", + // decimals: 18, + // }, + // explorers: [ + // { + // name: "etherscan-goerli", + // url: "https://goerli.etherscan.io", + // standard: "EIP3091", + // }, + // ], + // txConst: { + // blockTimeSec: 12, + // rbfTimes: 6, + // boardcastTimes: 5, + // }, + // contracts: { git3: "0x51bb7F23193b88696D25EAec7E3293a2C96e55Ee" }, + // }, 3334: { name: "Web3Q Galileo", nativeCurrency: { @@ -67,29 +67,54 @@ const evmNetworks: Record = { }, contracts: { git3: "0x59ef6b2dbfE86CcAaD84E2d8e78177f528521Da9" }, }, - 3141: { - name: "Filecoin - Hyperspace testnet", + // 3141: { + // name: "Filecoin - Hyperspace testnet", + // nativeCurrency: { + // name: "testnet filecoin", + // symbol: "tFIL", + // decimals: 18, + // }, + // rpc: [ + // "https://api.hyperspace.node.glif.io/rpc/v1", + // // "https://filecoin-hyperspace.chainstacklabs.com/rpc/v1", + // ], + // explorers: [ + // { + // name: "Filfox - Hyperspace", + // url: "https://hyperspace.filfox.info/en", + // standard: "none", + // }, + // ], + // txConst: { + // blockTimeSec: 30, + // boardcastTimes: 5, + // }, + // contracts: { git3: "0xF56A1dd941667911896B9B872AC79E56cfc6a3dB" }, + // }, + 421613: { + name: "Arbitrum - Goerli", nativeCurrency: { - name: "testnet filecoin", - symbol: "tFIL", + name: "Arbitrum Ether", + symbol: "ETH", decimals: 18, }, rpc: [ - "https://api.hyperspace.node.glif.io/rpc/v1", - // "https://filecoin-hyperspace.chainstacklabs.com/rpc/v1", + "https://goerli-rollup.arbitrum.io/rpc", + "https://arb-goerli.g.alchemy.com/v2/XT-0xyP5nTP1ltJ0MRBzvO-K9taRwt9o", ], explorers: [ { - name: "Filfox - Hyperspace", - url: "https://hyperspace.filfox.info/en", + name: "arbiscan-goerli", + url: "https://goerli.arbiscan.io", standard: "none", }, ], txConst: { - blockTimeSec: 30, - boardcastTimes: 5, + blockTimeSec: 3, + rbfTimes: 5, + boardcastTimes: 10, }, - contracts: { git3: "0xF56A1dd941667911896B9B872AC79E56cfc6a3dB" }, + contracts: { git3: "0x7Bb1038106fC6490195ec9906b29C81217ab090d" }, }, } diff --git a/src/config/ipfs.ts b/src/config/ipfs.ts index ae5fa75..74f726e 100644 --- a/src/config/ipfs.ts +++ b/src/config/ipfs.ts @@ -2,7 +2,6 @@ export default { gateways: [ "https://ipfs.io/ipfs/", "https://cloudflare-ipfs.com/ipfs/", - "https://ipfs.fleek.co/ipfs/", "https://gateway.ipfs.io/ipfs/", ], } diff --git a/src/config/name-services.ts b/src/config/name-services.ts index 6d83b5f..efe4866 100644 --- a/src/config/name-services.ts +++ b/src/config/name-services.ts @@ -15,6 +15,10 @@ const ns: Record = { chainId: 5, resolver: "0x00000000000C2E074eC69A0dFb2997BA6C7d2e1e", }, + arb: { + chainId: 421613, + resolver: "0x00000000000C2E074eC69A0dFb2997BA6C7d2e1e", + }, } export default ns diff --git a/src/storage/SLIStorage.ts b/src/storage/SLIStorage.ts index 13e50d9..a652689 100644 --- a/src/storage/SLIStorage.ts +++ b/src/storage/SLIStorage.ts @@ -4,14 +4,14 @@ import { ethers } from "ethers" import ipfsConf from "../config/ipfs.js" import axios from "axios" import { Git3Protocol } from "../common/git3-protocol.js" -import { QueueTask } from "../common/queue-task.js" +import { QueueTask, Retrier } from "../common/queue-task.js" export class SLIStorage implements Storage { repoName: string wallet: ethers.Wallet contract: ethers.Contract txManager: TxManager - auth: string + auth: string[] batchQueue: Record[] = [] maxBatchSize = 20 @@ -28,8 +28,10 @@ export class SLIStorage implements Storage { this.contract = protocol.contract this.wallet = protocol.wallet this.txManager = new TxManager(this.contract, protocol.chainId, protocol.netConfig.txConst) - this.auth = - "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJkaWQ6ZXRocjoweGFEQTdCOWFlQTdGNTc2ZDI5NzM0ZWUxY0Q2ODVFMzc2OWNCM2QwRDEiLCJpc3MiOiJuZnQtc3RvcmFnZSIsImlhdCI6MTY3NTQ5NDYwMDkzMiwibmFtZSI6ImZ2bS1oYWNrc29uIn0.YBqfsj_LTZSJPKc0OH586avnQNqove_Htzl5rrToXTk" + this.auth = [ + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJkaWQ6ZXRocjoweGFEQTdCOWFlQTdGNTc2ZDI5NzM0ZWUxY0Q2ODVFMzc2OWNCM2QwRDEiLCJpc3MiOiJuZnQtc3RvcmFnZSIsImlhdCI6MTY3NTQ5NDYwMDkzMiwibmFtZSI6ImZ2bS1oYWNrc29uIn0.YBqfsj_LTZSJPKc0OH586avnQNqove_Htzl5rrToXTk", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJkaWQ6ZXRocjoweDhmOTY1ZjAyRWY1MzkxODBlNDNiQ0M5M0FkZDJDZDI1RjU5RjRiMzIiLCJpc3MiOiJuZnQtc3RvcmFnZSIsImlhdCI6MTY3NjY1NDE1MzExMCwibmFtZSI6ImdpdDMifQ.f7vpBmQCMV3VIqWfPtuDNA5G5ThegjVaO4V-GCmK6wg", + ] this.txManager = new TxManager(this.contract, protocol.chainId, protocol.netConfig.txConst) @@ -38,9 +40,9 @@ export class SLIStorage implements Storage { }) this.storageTask = new QueueTask({ - maxRetry: 10, + maxRetry: 30, queueInterval: 400, - maxPending: 20, + maxPending: 100, retryInterval: 500, }) } @@ -57,19 +59,31 @@ export class SLIStorage implements Storage { } async download(path: string): Promise<[Status, Buffer]> { - const res = await this.contract.download(Buffer.from(this.repoName), Buffer.from(path)) + const res = await Retrier( + async () => await this.contract.download(Buffer.from(this.repoName), Buffer.from(path)), + { maxRetry: 10 } + ) const buffer = Buffer.from(res.slice(2), "hex") const cid = buffer.toString("utf8") for (let i = 0; i < ipfsConf.gateways.length; i++) { let gateway = ipfsConf.gateways[Math.floor(Math.random() * ipfsConf.gateways.length)] //random get rpc try { - let response = await axios.get(gateway + cid, { - responseType: "arraybuffer", - }) - if (response.status === 200) { - console.error(`=== download file ${path} succeed ===`) - return [Status.SUCCEED, Buffer.from(response.data)] - } + let resault = await Retrier( + async () => { + const TIMEOUT = 15 + let response = await axios.get(gateway + cid, { + responseType: "arraybuffer", + timeout: TIMEOUT * 1000, + }) + if (response.status === 200) { + return Buffer.from(response.data) + } else { + throw new Error(`download failed: ${response.status}`) + } + }, + { maxRetry: 3 } + ) + return [Status.SUCCEED, resault] } catch (e) { //pass } @@ -191,13 +205,13 @@ export class SLIStorage implements Storage { } async storeIPFS(data: Buffer): Promise { - const TIMEOUT = 30 + const TIMEOUT = 15 try { let cid = await this.storageTask.run(async () => { let response = await axios.post("https://api.nft.storage/upload", data, { headers: { "Content-Type": "application/octet-stream", - Authorization: this.auth, + Authorization: this.auth[Math.floor(Math.random() * this.auth.length)], }, timeout: TIMEOUT * 1000, })