diff --git a/lib/tasks/transcribe.js b/lib/tasks/transcribe.js
index a1d63b3a..aa415148 100644
--- a/lib/tasks/transcribe.js
+++ b/lib/tasks/transcribe.js
@@ -87,6 +87,7 @@ class TaskTranscribe extends SttTask {
 
   async exec(cs, obj) {
     try {
+      this.hasFallbackAsrFromLastRun = cs.hasFallbackAsr;
       await this.handling(cs, obj);
     } catch (error) {
       if (error instanceof SpeechCredentialError) {
@@ -905,10 +906,19 @@ class TaskTranscribe extends SttTask {
   // we want to keep track this on task level instead of endpoint level
   // because the endpoint instance is used across multiple tasks.
   canFallback(ep) {
-    const isHandledByPrimaryProvider = ep === this.ep ?
-      this.isHandledByPrimaryProviderForEp1 : ep === this.ep2 ?
-        this.isHandledByPrimaryProviderForEp2 : this.isHandledByPrimaryProvider;
-    return this.fallbackVendor && isHandledByPrimaryProvider && !this.cs.hasFallbackAsr;
+    let isHandledByPrimaryProvider = this.isHandledByPrimaryProvider;
+    if (ep === this.ep) {
+      isHandledByPrimaryProvider = this.isHandledByPrimaryProviderForEp1;
+    } else if (ep === this.ep2) {
+      isHandledByPrimaryProvider = this.isHandledByPrimaryProviderForEp2;
+    }
+
+    // fallback is configured
+    return this.fallbackVendor &&
+      // has this endpoint already fallen back
+      isHandledByPrimaryProvider &&
+      // in global level, is there any fallback is already happened
+      !this.hasFallbackAsrFromLastRun;
   }
 
   _initFallback(ep) {