When requests to Step Function fail, Workflow Service performs exponential backoff retries to notify Step Function service. Number of retries = 3 The total duration of all attempts doesn't exceed 10 seconds.

@Override
        public void onFailure(Throwable t) {
            if (numOfRetries < MAX_RETRIES) {
                logger.info("Retrying the action for the " + numOfRetries + " time.");
                numOfRetries++;
                executorService.schedule(() -> {
                    PublishMetricsAction.FailedMessageRetry failedMessageRetry = new PublishMetricsAction.FailedMessageRetry(t);
                    failedMessageRetry.execute();
                }, getNextBackOff(numOfRetries));
            } else {
                logger.error("Failed to publish metrics after " + numOfRetries + " retries.");
                //TODO: send error email
            }
        }

        private long getNextBackOff(int numOfRetries){
            if (numOfRetries < 0 || numOfRetries >= MAX_RETRIES) {
                return 0;
            }
            double exponentialBackOffMs = retryBackOff[numOfRetries] * Math.pow(2, numOfRetries