diff --git a/lib/tasks/transcribe.js b/lib/tasks/transcribe.js
index aa415148..4ae379f4 100644
--- a/lib/tasks/transcribe.js
+++ b/lib/tasks/transcribe.js
@@ -87,7 +87,7 @@ class TaskTranscribe extends SttTask {
 
   async exec(cs, obj) {
     try {
-      this.hasFallbackAsrFromLastRun = cs.hasFallbackAsr;
+      this.hasFallbackAsrFromStart = cs.hasFallbackAsr;
       await this.handling(cs, obj);
     } catch (error) {
       if (error instanceof SpeechCredentialError) {
@@ -913,12 +913,17 @@ class TaskTranscribe extends SttTask {
       isHandledByPrimaryProvider = this.isHandledByPrimaryProviderForEp2;
     }
 
+    const isOneOfEndpointAlreadyFallenBack =
+      this.isHandledByPrimaryProviderForEp1 !== this.isHandledByPrimaryProviderForEp2;
+
     // fallback is configured
     return this.fallbackVendor &&
       // has this endpoint already fallen back
       isHandledByPrimaryProvider &&
       // in global level, is there any fallback is already happened
-      !this.hasFallbackAsrFromLastRun;
+      // one fallen endpoint will mark cs.hasFallbackAsr to true,
+      // so the other endpoint will be able to fallback again
+      (isOneOfEndpointAlreadyFallenBack || !this.cs.hasFallbackAsr);
   }
 
   _initFallback(ep) {