HTTP POST outcalls consensus not working , only one is working

So basically this is my code , which implements the following code

  ic_cdk::print("eth main_task");
    let latestBLockNumber = LASTESET_BLOCK_READ.with(|block| (*block.borrow()).clone());
    // let latestBlockNum64 =// Candid Nat as a string
    let block_big_int = latestBLockNumber.0.clone();
    let block_uint =  u64::try_from(block_big_int.clone()).unwrap();
    let block_string = format!("0x{:0x}", block_uint);
    ic_cdk::println!(" eth block string poly {}",block_string);
    ic_cdk::println!("eth block_uint {}",block_uint);
    let correctBlock = if Nat::from(0) == latestBLockNumber  {"earliest"} else { &block_string };
    // let body = "{\"jsonrpc\": \"2.0\",\"method\": \"eth_getLogs\",\"params\": [{\"fromBlock\": \"earliest\",\"toBlock\": \"latest\",\"address\":\"0xe7399b79838acc8caaa567fF84e5EFd0d11BB010\",\"topics\":[\"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef\"]}],\"id\": 1}";
    // let unformated_body: &str = "{\"jsonrpc\": \"2.0\",\"method\": \"eth_getLogs\",\"params\": [{\"fromBlock\": \"#\",\"toBlock\": \"latest\",\"address\":\"0xe7399b79838acc8caaa567fF84e5EFd0d11BB010\",\"topics\":[\"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef\"]}],\"id\": 1}";
    let unformated_body: &str = "{\"jsonrpc\": \"2.0\",\"method\": \"eth_getLogs\",\"params\": [{\"fromBlock\": \"#\",\"toBlock\": \"latest\",\"address\":\"0x5E906C9f094906c80F34e8524C8Eec81D19CEcD2\",\"topics\":[\"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef\"]}],\"id\": 1}";
    let body = unformated_body.replace("#",&correctBlock);
    ic_cdk::println!("eth body: {}", body);
    // rpc_call("{\"jsonrpc\": \"2.0\",\"method\": \"eth_getLogs\",\"params\": [{\"fromBlock\": \"earliest\",\"toBlock\": \"latest\",\"address\":\"0xe7399b79838acc8caaa567fF84e5EFd0d11BB010\",\"topics\":[\"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef\"]}],\"id\": 1}").await()
    let w3 = match ICHttp::new(URL_POLYGON, None) {
        Ok(v) => { Web3::new(v) },
        Err(e) => { return Err(e.to_string()) },

    let res = w3.json_rpc_call(body.as_ref()).await.map_err(|e| format!("{}", e))?;
    ic_cdk::println!("res: {}", res);
    // ic_cdk::println!("jsonString: {}", jsonString);
    let logResponse:Vec<EventResult> = serde_json::from_str(&res).unwrap();
        ic_cdk::println!("eth no transaction at all");
        return Ok("no new block".to_string());
    let lastIndex : usize = logResponse.len() - 1;
    let blockNumberInHex = &logResponse[lastIndex].block_number;
    let withoutPrefixBloackNumberInHex = blockNumberInHex.trim_start_matches("0x");
    let lastestBlack  = u64::from_str_radix(withoutPrefixBloackNumberInHex, 16).unwrap(); 
    let hex_value = &logResponse[lastIndex].data;
    let value: U256 = U256::from_str_radix(&hex_value[2..], 16).unwrap();
    ic_cdk::println!("-----------------------------------value-eth---------------- :{}",value);
    ic_cdk::println!("lastestBlack: {}", lastestBlack);
    ic_cdk::println!("latestBLockNumber of application state: {}", latestBLockNumber);
    ic_cdk::println!(" boolean: {}", Nat::from(lastestBlack) > latestBLockNumber);
        ic_cdk::println!("eth no new block");
        return Ok("no new block".to_string());
    let lastest_tx_hash=&logResponse[lastIndex].transaction_hash; 
    let tx_body: &str = "{\"jsonrpc\":\"2.0\",\"method\":\"eth_getTransactionByHash\",\"params\": [\"#\"],\"id\":1}";
    let getTransactionBody =tx_body.replace("#", &*lastest_tx_hash);
    ic_cdk::println!("Eth getTransactionBody: {}", getTransactionBody);
    let getTransactionRes = w3.json_rpc_call(&getTransactionBody).await.map_err(|e| format!(" failed in transaction history{}", e))?;
    let transaction: Transaction = serde_json::from_str(&getTransactionRes).unwrap();
    let methodId = &transaction.input[0..10];
    let adminMethodId = "0x0d271720";
    ic_cdk::println!("Eth transaction: {}", getTransactionRes);
        ic_cdk::println!(" eth adminMethodId: {}", adminMethodId);
        LASTESET_BLOCK_READ.with(|v| *v.borrow_mut() = Nat::from(lastestBlack));
        return Ok("This is the admin function".to_string());
    if (Nat::from(lastestBlack) > latestBLockNumber ){
        let latestBlock: &EventResult = &logResponse[lastIndex];
        ic_cdk::println!("eth new block found");
         // ecdsa key info
        let derivation_path = vec![ic_cdk::id().as_slice().to_vec()];
        let key_info = KeyInfo{ derivation_path: derivation_path, key_name: KEY_NAME.to_string(), ecdsa_sign_cycles: None };
        //from address
        let raw_from = &latestBlock.topics[1];
        let from = Address::from_str(&get_address_from_topic(&raw_from)).unwrap();
        ic_cdk::println!("eth from----------------------------: {}", from);
        let w3 = match ICHttp::new(URL, None) {
            Ok(v) => { Web3::new(v) },
            Err(e) => { return Err(e.to_string()) },
        //contract address
        // let poly_contract_address = "0xe7399b79838acc8caaa567fF84e5EFd0d11BB010";
        let poly_contract_address = "0xe5cdFC9a5A59E9949f6C31aB05De8d7DB414756F";
        let contract_address = Address::from_str(&poly_contract_address).unwrap();
        let contract = Contract::from_json(w3.eth(),contract_address, TOKEN_ABI).map_err(|e| format!("init contract failed: {}", e))?;

        let canister_addr = get_eth_addr(None, None, KEY_NAME.to_string()).await.map_err(|e| format!("get canister eth addr failed: {}", e))?;
        // add nonce to options
            let tx_count = w3.eth()
                .transaction_count(canister_addr, None)
                .map_err(|e| format!("get tx count error: {}", e))?;
        let gas_price = w3.eth()
        .map_err(|e| format!("get gas_price error: {}", e))?;
        // legacy transaction type is still ok
        let options = Options::with(|op| { 
            op.nonce = Some(tx_count);
            op.gas_price = Some(gas_price);
            op.transaction_type = Some(U64::from(2)) //EIP1559_TX_ID
        let raw_to = &latestBlock.topics[2];
        let to_addr = Address::from_str(&get_address_from_topic(&raw_to)).unwrap();
        ic_cdk::println!("to_addr-----------------------------: {}", to_addr);
        LASTESET_BLOCK_READ.with(|v| *v.borrow_mut() = Nat::from(lastestBlack));
        let txhash = contract
            .signed_call("transferFromAdmin", (from,to_addr, value,), options, hex::encode(canister_addr), key_info, CHAIN_ID)
            .map_err(|e| format!("token transfer failed: {}", e))?;
        ic_cdk::println!("eth txhash: {}", hex::encode(txhash));
        ic_cdk::println!("eth no new block");
        Ok("no new block".to_string())

This code is in a ic_timer function and runs every 10 seconds
as you can see i am using Eth_logs , eth_getTransactionByHash and sendRawTransaction , and all these are POST methods , but for some reason , only Eth_logs are working , i know this by checking my alchemy logs .

This is the weird part , why is eth_logs working and not the rest.

And finally i have some candid functions like getGasPrice , sendEth all these using POST JSON RPC Http out calls , and i am getting this error

(variant {Err="get tx count error: The http_request resulted into error. RejectionCode: SysTransient, Error: Canister http responses were different across replicas, and no consensus was reached"})

Because the ETH RPC is not idempotent. Internet Computer Loading

Idempotency , is it something that should be implemented on server side ? Or canister side ? Plus why is it that eth_logs was called ? If the http post outcalls were not idempotent

Also do you know any other node providers that do support idempotency ?

Yes that needs to be implemented server-side

Currently what you can do with Alchemy is to buy the credits, and modify your canister code to catch the error and continue.

thats not possible , those http outcalls are mission critical

How would buying alchemy credits solve it ?

Some free RPC services have call limits.

Well if you check my eth_logs are working fine in free tier so I don’t think that’s the issue

How about I try this

  1. I create my own idempotent ipv6 compatible express js backend
  2. The express js backend wraps arround the alchemy node provider

Will this stop the idempotency issue . ?

Yes, that will work. Of course you then have another single point of failure, but you have that anyways with alchemy

We same issue with every other blockchain ever apart from ICP that doesn’t need a thrid party node provider