fix(data): fix the incorrect output of scipy.stft() data to be pandas.DataFrame shaped (513,513) along with its frequencies as the index and times as the columns (transposed) instead of just the magnitude that being flattened out; add checks for empty data and correct file paths for sensor data loading.

Closes #43

code/notebooks/stft.ipynb

@@ -121,6 +121,7 @@
     "signal_sensor2_test1 = []\n",
     "\n",
     "for data in df:\n",
+    "    if not data.empty and 'sensor 1' in data.columns and 'sensor 2' in data.columns:\n",
     "        signal_sensor1_test1.append(data['sensor 1'].values)\n",
     "        signal_sensor2_test1.append(data['sensor 2'].values)\n",
     "\n",
@@ -156,8 +157,6 @@
     "from scipy.signal import stft, hann\n",
     "from multiprocessing import Pool\n",
     "\n",
-    "\n",
-    "\n",
     "# Function to compute and append STFT data\n",
     "def process_stft(args):\n",
     "    # Define STFT parameters\n",
@@ -199,23 +198,22 @@
     "    # Compute STFT\n",
     "    frequencies, times, Zxx = stft(sensor_data, fs=Fs, window=window, nperseg=window_size, noverlap=window_size - hop_size)\n",
     "    magnitude = np.abs(Zxx)\n",
-    "    flattened_stft = magnitude.flatten()\n",
+    "    df_stft = pd.DataFrame(magnitude, index=frequencies, columns=times).T\n",
+    "    df_stft.columns = [f\"Freq_{i}\" for i in frequencies]\n",
     "    \n",
     "    # Define the output CSV file path\n",
     "    stft_file_name = f'stft_data{sensor_num}_{damage_num}.csv'\n",
     "    sensor_output_dir = os.path.join(damage_base_path, sensor_name.lower())\n",
     "    os.makedirs(sensor_output_dir, exist_ok=True)\n",
     "    stft_file_path = os.path.join(sensor_output_dir, stft_file_name)\n",
-    "    print(stft_file_path)\n",
     "    # Append the flattened STFT to the CSV\n",
     "    try:\n",
-    "        flattened_stft_df = pd.DataFrame([flattened_stft])\n",
     "        if not os.path.isfile(stft_file_path):\n",
     "            # Create a new CSV\n",
-    "            flattened_stft_df.to_csv(stft_file_path, index=False, header=False)\n",
+    "            df_stft.to_csv(stft_file_path, index=False, header=False)\n",
     "        else:\n",
     "            # Append to existing CSV\n",
-    "            flattened_stft_df.to_csv(stft_file_path, mode='a', index=False, header=False)\n",
+    "            df_stft.to_csv(stft_file_path, mode='a', index=False, header=False)\n",
     "        print(f\"Appended STFT data to {stft_file_path}\")\n",
     "    except Exception as e:\n",
     "        print(f\"Error writing to {stft_file_path}: {e}\")"
@@ -295,7 +293,7 @@
     "\n",
     "# get current y ticks in list\n",
     "print(len(frequencies))\n",
-    "print(len(times))\n"
+    "print(len(times))"
    ]
   },
   {
@@ -324,8 +322,8 @@
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
     "ready_data1 = []\n",
-    "for file in os.listdir('D:/thesis/data/working/sensor1'):\n",
+    "for file in os.listdir('D:/thesis/data/converted/raw/sensor1'):\n",
-    "    ready_data1.append(pd.read_csv(os.path.join('D:/thesis/data/working/sensor1', file)))\n",
+    "    ready_data1.append(pd.read_csv(os.path.join('D:/thesis/data/converted/raw/sensor1', file)))\n",
     "# ready_data1[1]\n",
     "# colormesh give title x is frequency and y is time and rotate/transpose the data\n",
     "# Plotting the STFT Data"
@@ -337,8 +335,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ready_data1[1]\n",
+    "# ready_data1[1]\n",
-    "plt.pcolormesh(ready_data1[1])"
+    "plt.pcolormesh(ready_data1[2])"
    ]
   },
   {
@@ -362,9 +360,8 @@
    "outputs": [],
    "source": [
     "ready_data2 = []\n",
-    "for file in os.listdir('D:/thesis/data/working/sensor2'):\n",
+    "for file in os.listdir('D:/thesis/data/converted/raw/sensor2'):\n",
-    "    ready_data2.append(pd.read_csv(os.path.join('D:/thesis/data/working/sensor2', file)))\n",
+    "    ready_data2.append(pd.read_csv(os.path.join('D:/thesis/data/converted/raw/sensor2', file)))"
-    "ready_data2[5]"
    ]
   },
   {
@@ -384,10 +381,25 @@
    "outputs": [],
    "source": [
     "x1 = 0\n",
-    "\n",
+    "print(type(ready_data1[0]))\n",
+    "ready_data1[0].iloc[:,0]\n",
+    "# x1 = x1 + ready_data1[0].shape[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "x1 = 0\n",
+    "print(type(x1))\n",
     "for i in range(len(ready_data1)):\n",
-    "    print(ready_data1[i].shape)\n",
+    "    # print(ready_data1[i].shape)\n",
+    "    # print(ready_data1[i].)\n",
+    "    print(type(ready_data1[i].shape[0]))\n",
     "    x1 = x1 + ready_data1[i].shape[0]\n",
+    "    print(type(x1))\n",
     "\n",
     "print(x1)"
    ]