fix(dashboard): Handle encoding errors (#3847)

dashboard/config.py

@@ -1,4 +1,5 @@
 import os
+import sys
 
 # Emojis to be used in the compliance table
 pass_emoji = "✅"
@@ -28,5 +29,10 @@ informational_color = "#3274d9"
 folder_path_overview = os.getcwd() + "/output"
 folder_path_compliance = os.getcwd() + "/output/compliance"
 
-# Encoding
-encoding_format = "utf-8"
+# Encoding, if the os is windows, use cp1252. Use utf-8 if it is running using python3
+if os.name == "nt" and ".py" not in sys.argv[0].lower():
+    encoding_format = "cp1252"
+else:
+    encoding_format = "utf-8"
+# Error action, it is recommended to use "ignore" or "replace"
+error_action = "ignore"

dashboard/pages/compliance.py

@@ -16,6 +16,7 @@ from dash.dependencies import Input, Output
 # Config import
 from dashboard.config import (
     encoding_format,
+    error_action,
     fail_color,
     folder_path_compliance,
     info_color,
@@ -29,6 +30,7 @@ from dashboard.lib.dropdowns import (
     create_region_dropdown_compliance,
 )
 from dashboard.lib.layouts import create_layout_compliance
+from prowler.lib.logger import logger
 
 # Suppress warnings
 warnings.filterwarnings("ignore")
@@ -38,11 +40,16 @@ warnings.filterwarnings("ignore")
 
 csv_files = []
 for file in glob.glob(os.path.join(folder_path_compliance, "*.csv")):
-    with open(file, "r", newline="", encoding=encoding_format) as csvfile:
-        reader = csv.reader(csvfile)
-        num_rows = sum(1 for row in reader)
-        if num_rows > 1:
-            csv_files.append(file)
+    try:
+        with open(
+            file, "r", newline="", encoding=encoding_format, errors=error_action
+        ) as csvfile:
+            reader = csv.reader(csvfile)
+            num_rows = sum(1 for row in reader)
+            if num_rows > 1:
+                csv_files.append(file)
+    except UnicodeDecodeError:
+        logger.error(f"Error decoding file: {file}")
 
 
 def load_csv_files(csv_files):
@@ -50,7 +57,7 @@ def load_csv_files(csv_files):
     dfs = []
     results = []
     for file in csv_files:
-        df = pd.read_csv(file, sep=";", on_bad_lines="skip")
+        df = pd.read_csv(file, sep=";", on_bad_lines="skip", encoding=encoding_format)
         if "CHECKID" in df.columns:
             dfs.append(df)
             result = file
@@ -238,7 +245,9 @@ def display_data(
         """Load CSV files into a single pandas DataFrame."""
         dfs = []
         for file in files:
-            df = pd.read_csv(file, sep=";", on_bad_lines="skip")
+            df = pd.read_csv(
+                file, sep=";", on_bad_lines="skip", encoding=encoding_format
+            )
             dfs.append(df.astype(str))
         return pd.concat(dfs, ignore_index=True)
 

dashboard/pages/overview.py

@@ -18,6 +18,7 @@ from dash.dependencies import Input, Output
 from dashboard.config import (
     critical_color,
     encoding_format,
+    error_action,
     fail_color,
     folder_path_overview,
     high_color,
@@ -42,6 +43,7 @@ from dashboard.lib.dropdowns import (
     create_table_row_dropdown,
 )
 from dashboard.lib.layouts import create_layout_overview
+from prowler.lib.logger import logger
 
 # Suppress warnings
 warnings.filterwarnings("ignore")
@@ -51,11 +53,16 @@ warnings.filterwarnings("ignore")
 csv_files = []
 
 for file in glob.glob(os.path.join(folder_path_overview, "*.csv")):
-    with open(file, "r", newline="", encoding=encoding_format) as csvfile:
-        reader = csv.reader(csvfile)
-        num_rows = sum(1 for row in reader)
-        if num_rows > 1:
-            csv_files.append(file)
+    with open(
+        file, "r", newline="", encoding=encoding_format, errors=error_action
+    ) as csvfile:
+        try:
+            reader = csv.reader(csvfile)
+            num_rows = sum(1 for row in reader)
+            if num_rows > 1:
+                csv_files.append(file)
+        except UnicodeDecodeError:
+            logger.error(f"Error decoding file: {file}")
 
 
 # Import logos providers
@@ -77,7 +84,7 @@ def load_csv_files(csv_files):
     """Load CSV files into a single pandas DataFrame."""
     dfs = []
     for file in csv_files:
-        df = pd.read_csv(file, sep=";", on_bad_lines="skip")
+        df = pd.read_csv(file, sep=";", on_bad_lines="skip", encoding=encoding_format)
         if "CHECK_ID" in df.columns:
             if "TIMESTAMP" in df.columns or df["PROVIDER"].unique() == "aws":
                 dfs.append(df.astype(str))
@@ -456,7 +463,7 @@ def filter_data(
     # Select the files in the list_files that have the same date as the selected date
     list_files = []
     for file in csv_files:
-        df = pd.read_csv(file, sep=";", on_bad_lines="skip")
+        df = pd.read_csv(file, sep=";", on_bad_lines="skip", encoding=encoding_format)
         if "CHECK_ID" in df.columns:
             if "TIMESTAMP" in df.columns or df["PROVIDER"].unique() == "aws":
                 # This handles the case where we are using v3 outputs