nuluh/thesis
1
0
Fork 0
Code Issues 47 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(data): fix the incorrect output of scipy.stft() data to be pandas.DataFrame shaped (513,513) along with its frequencies as the index and times as the columns (transposed) instead of just the magnitude that being flattened out; add checks for empty data and correct file paths for sensor data loading.

Browse Source 
Closes #43
This commit is contained in:
nuluh
2025-04-20 14:45:38 +07:00
parent 36b36c41ba
commit db2947abdf
1 changed files with 31 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
code/notebooks/stft.ipynb
View File

@@ -121,6 +121,7 @@
"signal_sensor2_test1 = []\n",
"\n",
"for data in df:\n",
" if not data.empty and 'sensor 1' in data.columns and 'sensor 2' in data.columns:\n",
" signal_sensor1_test1.append(data['sensor 1'].values)\n",
" signal_sensor2_test1.append(data['sensor 2'].values)\n",
"\n",
@@ -156,8 +157,6 @@
"from scipy.signal import stft, hann\n",
"from multiprocessing import Pool\n",
"\n",
"\n",
"\n",
"# Function to compute and append STFT data\n",
"def process_stft(args):\n",
" # Define STFT parameters\n",
@@ -199,23 +198,22 @@
" # Compute STFT\n",
" frequencies, times, Zxx = stft(sensor_data, fs=Fs, window=window, nperseg=window_size, noverlap=window_size - hop_size)\n",
" magnitude = np.abs(Zxx)\n",
" flattened_stft = magnitude.flatten()\n",
" df_stft = pd.DataFrame(magnitude, index=frequencies, columns=times).T\n",
" df_stft.columns = [f\"Freq_{i}\" for i in frequencies]\n",
" \n",
" # Define the output CSV file path\n",
" stft_file_name = f'stft_data{sensor_num}_{damage_num}.csv'\n",
" sensor_output_dir = os.path.join(damage_base_path, sensor_name.lower())\n",
" os.makedirs(sensor_output_dir, exist_ok=True)\n",
" stft_file_path = os.path.join(sensor_output_dir, stft_file_name)\n",
" print(stft_file_path)\n",
" # Append the flattened STFT to the CSV\n",
" try:\n",
" flattened_stft_df = pd.DataFrame([flattened_stft])\n",
" if not os.path.isfile(stft_file_path):\n",
" # Create a new CSV\n",
" flattened_stft_df.to_csv(stft_file_path, index=False, header=False)\n",
" df_stft.to_csv(stft_file_path, index=False, header=False)\n",
" else:\n",
" # Append to existing CSV\n",
" flattened_stft_df.to_csv(stft_file_path, mode='a', index=False, header=False)\n",
" df_stft.to_csv(stft_file_path, mode='a', index=False, header=False)\n",
" print(f\"Appended STFT data to {stft_file_path}\")\n",
" except Exception as e:\n",
" print(f\"Error writing to {stft_file_path}: {e}\")"
@@ -295,7 +293,7 @@
"\n",
"# get current y ticks in list\n",
"print(len(frequencies))\n",
"print(len(times))\n"
"print(len(times))"
]
},
{
@@ -324,8 +322,8 @@
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"ready_data1 = []\n",
"for file in os.listdir('D:/thesis/data/working/sensor1'):\n",
" ready_data1.append(pd.read_csv(os.path.join('D:/thesis/data/working/sensor1', file)))\n",
"for file in os.listdir('D:/thesis/data/converted/raw/sensor1'):\n",
" ready_data1.append(pd.read_csv(os.path.join('D:/thesis/data/converted/raw/sensor1', file)))\n",
"# ready_data1[1]\n",
"# colormesh give title x is frequency and y is time and rotate/transpose the data\n",
"# Plotting the STFT Data"
@@ -337,8 +335,8 @@
"metadata": {},
"outputs": [],
"source": [
"ready_data1[1]\n",
"plt.pcolormesh(ready_data1[1])"
"# ready_data1[1]\n",
"plt.pcolormesh(ready_data1[2])"
]
},
{
@@ -362,9 +360,8 @@
"outputs": [],
"source": [
"ready_data2 = []\n",
"for file in os.listdir('D:/thesis/data/working/sensor2'):\n",
" ready_data2.append(pd.read_csv(os.path.join('D:/thesis/data/working/sensor2', file)))\n",
"ready_data2[5]"
"for file in os.listdir('D:/thesis/data/converted/raw/sensor2'):\n",
" ready_data2.append(pd.read_csv(os.path.join('D:/thesis/data/converted/raw/sensor2', file)))"
]
},
{
@@ -384,10 +381,25 @@
"outputs": [],
"source": [
"x1 = 0\n",
"\n",
"print(type(ready_data1[0]))\n",
"ready_data1[0].iloc[:,0]\n",
"# x1 = x1 + ready_data1[0].shape[0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"x1 = 0\n",
"print(type(x1))\n",
"for i in range(len(ready_data1)):\n",
" print(ready_data1[i].shape)\n",
" # print(ready_data1[i].shape)\n",
" # print(ready_data1[i].)\n",
" print(type(ready_data1[i].shape[0]))\n",
" x1 = x1 + ready_data1[i].shape[0]\n",
" print(type(x1))\n",
"\n",
"print(x1)"
]