From eb62c7e61403291e1c983c7dec8c94a591eed8a8 Mon Sep 17 00:00:00 2001 From: nuluh Date: Thu, 24 Apr 2025 10:21:07 +0700 Subject: [PATCH] feat(notebook): Update variable names for clarity, remove unused imports, and streamline data processing. Implement data concatenation using pandas concat for efficiency. Add validation steps for Dataset B and improve model training consistency across sensors. --- .vscode/settings.json | 3 +- code/notebooks/stft.ipynb | 249 +++++++++++++++++++++++++------------- 2 files changed, 164 insertions(+), 88 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index a1299c3..a8b3783 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,4 @@ { - "python.analysis.extraPaths": ["./code/src/features"] + "python.analysis.extraPaths": ["./code/src/features"], + "jupyter.notebookFileRoot": "${workspaceFolder}/code" } diff --git a/code/notebooks/stft.ipynb b/code/notebooks/stft.ipynb index c8ef848..41137d9 100644 --- a/code/notebooks/stft.ipynb +++ b/code/notebooks/stft.ipynb @@ -155,7 +155,7 @@ "import pandas as pd\n", "import numpy as np\n", "from scipy.signal import stft, hann\n", - "from multiprocessing import Pool\n", + "# from multiprocessing import Pool\n", "\n", "# Function to compute and append STFT data\n", "def process_stft(args):\n", @@ -321,9 +321,9 @@ "source": [ "import pandas as pd\n", "import matplotlib.pyplot as plt\n", - "ready_data1 = []\n", + "ready_data1a = []\n", "for file in os.listdir('D:/thesis/data/converted/raw/sensor1'):\n", - " ready_data1.append(pd.read_csv(os.path.join('D:/thesis/data/converted/raw/sensor1', file)))\n", + " ready_data1a.append(pd.read_csv(os.path.join('D:/thesis/data/converted/raw/sensor1', file)))\n", "# colormesh give title x is frequency and y is time and rotate/transpose the data\n", "# Plotting the STFT Data" ] @@ -334,8 +334,8 @@ "metadata": {}, "outputs": [], "source": [ - "ready_data1[0]\n", - "plt.pcolormesh(ready_data1[0])" + "len(ready_data1a)\n", + "# plt.pcolormesh(ready_data1[0])" ] }, { @@ -345,7 +345,7 @@ "outputs": [], "source": [ "for i in range(6):\n", - " plt.pcolormesh(ready_data1[i])\n", + " plt.pcolormesh(ready_data1a[i])\n", " plt.title(f'STFT Magnitude for case {i} sensor 1')\n", " plt.xlabel(f'Frequency [Hz]')\n", " plt.ylabel(f'Time [sec]')\n", @@ -358,9 +358,9 @@ "metadata": {}, "outputs": [], "source": [ - "ready_data2 = []\n", + "ready_data2a = []\n", "for file in os.listdir('D:/thesis/data/converted/raw/sensor2'):\n", - " ready_data2.append(pd.read_csv(os.path.join('D:/thesis/data/converted/raw/sensor2', file)))" + " ready_data2a.append(pd.read_csv(os.path.join('D:/thesis/data/converted/raw/sensor2', file)))" ] }, { @@ -369,8 +369,8 @@ "metadata": {}, "outputs": [], "source": [ - "print(len(ready_data1))\n", - "print(len(ready_data2))" + "print(len(ready_data1a))\n", + "print(len(ready_data2a))" ] }, { @@ -379,10 +379,16 @@ "metadata": {}, "outputs": [], "source": [ - "x1 = 0\n", - "print(type(ready_data1[0]))\n", - "ready_data1[0].iloc[:,0]\n", - "# x1 = x1 + ready_data1[0].shape[0]" + "x1a = 0\n", + "print(type(ready_data1a[0]))\n", + "ready_data1a[0].iloc[:,0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Checking length of the total array" ] }, { @@ -391,16 +397,14 @@ "metadata": {}, "outputs": [], "source": [ - "x1 = 0\n", - "print(type(x1))\n", - "for i in range(len(ready_data1)):\n", - " # print(ready_data1[i].shape)\n", - " # print(ready_data1[i].)\n", - " print(type(ready_data1[i].shape[0]))\n", - " x1 = x1 + ready_data1[i].shape[0]\n", - " print(type(x1))\n", + "x1a = 0\n", + "print(type(x1a))\n", + "for i in range(len(ready_data1a)):\n", + " print(type(ready_data1a[i].shape[0]))\n", + " x1a = x1a + ready_data1a[i].shape[0]\n", + " print(type(x1a))\n", "\n", - "print(x1)" + "print(x1a)" ] }, { @@ -409,13 +413,20 @@ "metadata": {}, "outputs": [], "source": [ - "x2 = 0\n", + "x2a = 0\n", "\n", - "for i in range(len(ready_data2)):\n", - " print(ready_data2[i].shape)\n", - " x2 = x2 + ready_data2[i].shape[0]\n", + "for i in range(len(ready_data2a)):\n", + " print(ready_data2a[i].shape)\n", + " x2a = x2a + ready_data2a[i].shape[0]\n", "\n", - "print(x2)" + "print(x2a)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Flatten 6 array into one array" ] }, { @@ -424,28 +435,22 @@ "metadata": {}, "outputs": [], "source": [ - "x1 = ready_data1[0]\n", - "# print(x1)\n", - "print(type(x1))\n", - "for i in range(len(ready_data1) - 1):\n", - " #print(i)\n", - " x1 = np.concatenate((x1, ready_data1[i + 1]), axis=0)\n", - "# print(x1)\n", - "pd.DataFrame(x1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "x2 = ready_data2[0]\n", + "# Combine all dataframes in ready_data1a into a single dataframe\n", + "if ready_data1a: # Check if the list is not empty\n", + " # Use pandas concat function instead of iterative concatenation\n", + " combined_data = pd.concat(ready_data1a, axis=0, ignore_index=True)\n", + " \n", + " print(f\"Type of combined data: {type(combined_data)}\")\n", + " print(f\"Shape of combined data: {combined_data.shape}\")\n", + " \n", + " # Display the combined dataframe\n", + " combined_data\n", + "else:\n", + " print(\"No data available in ready_data1a list\")\n", + " combined_data = pd.DataFrame()\n", "\n", - "for i in range(len(ready_data2) - 1):\n", - " #print(i)\n", - " x2 = np.concatenate((x2, ready_data2[i + 1]), axis=0)\n", - "pd.DataFrame(x2)" + "# Store the result in x1a for compatibility with subsequent code\n", + "x1a = combined_data" ] }, { @@ -454,20 +459,29 @@ "metadata": {}, "outputs": [], "source": [ - "print(x1.shape)\n", - "print(x2.shape)" + "# Combine all dataframes in ready_data1a into a single dataframe\n", + "if ready_data2a: # Check if the list is not empty\n", + " # Use pandas concat function instead of iterative concatenation\n", + " combined_data = pd.concat(ready_data2a, axis=0, ignore_index=True)\n", + " \n", + " print(f\"Type of combined data: {type(combined_data)}\")\n", + " print(f\"Shape of combined data: {combined_data.shape}\")\n", + " \n", + " # Display the combined dataframe\n", + " combined_data\n", + "else:\n", + " print(\"No data available in ready_data1a list\")\n", + " combined_data = pd.DataFrame()\n", + "\n", + "# Store the result in x1a for compatibility with subsequent code\n", + "x2a = combined_data" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "y_1 = [1,1,1,1]\n", - "y_2 = [0,1,1,1]\n", - "y_3 = [1,0,1,1]\n", - "y_4 = [1,1,0,0]" + "### Creating the label" ] }, { @@ -490,7 +504,8 @@ "metadata": {}, "outputs": [], "source": [ - "y_data = [y_1, y_2, y_3, y_4, y_5, y_6]" + "y_data = [y_1, y_2, y_3, y_4, y_5, y_6]\n", + "y_data" ] }, { @@ -500,7 +515,7 @@ "outputs": [], "source": [ "for i in range(len(y_data)):\n", - " print(ready_data1[i].shape[0])" + " print(ready_data1a[i].shape[0])" ] }, { @@ -509,9 +524,9 @@ "metadata": {}, "outputs": [], "source": [ + "import numpy as np\n", "for i in range(len(y_data)):\n", - " y_data[i] = [y_data[i]]*ready_data1[i].shape[0]\n", - " y_data[i] = np.array(y_data[i])" + " y_data[i] = [y_data[i]]*ready_data1a[i].shape[0]" ] }, { @@ -520,6 +535,7 @@ "metadata": {}, "outputs": [], "source": [ + "# len(y_data[0])\n", "y_data" ] }, @@ -552,10 +568,10 @@ "metadata": {}, "outputs": [], "source": [ - "from sklearn.model_selection import train_test_split\n", + "from src.ml.model_selection import create_ready_data\n", "\n", - "x_train1, x_test1, y_train, y_test = train_test_split(x1, y, test_size=0.2, random_state=2)\n", - "x_train2, x_test2, y_train, y_test = train_test_split(x2, y, test_size=0.2, random_state=2)" + "X1a, y = create_ready_data('D:/thesis/data/converted/raw/sensor1')\n", + "X2a, y = create_ready_data('D:/thesis/data/converted/raw/sensor2')" ] }, { @@ -565,6 +581,17 @@ "outputs": [], "source": [ "from sklearn.model_selection import train_test_split\n", + "\n", + "x_train1, x_test1, y_train, y_test = train_test_split(X1a, y, test_size=0.2, random_state=2)\n", + "x_train2, x_test2, y_train, y_test = train_test_split(X2a, y, test_size=0.2, random_state=2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "from sklearn.metrics import accuracy_score\n", "from sklearn.ensemble import RandomForestClassifier, BaggingClassifier\n", "from sklearn.tree import DecisionTreeClassifier\n", @@ -597,16 +624,17 @@ "\n", "\n", "# 1. Random Forest\n", - "rf_model = RandomForestClassifier()\n", - "rf_model.fit(x_train1, y_train)\n", - "rf_pred1 = rf_model.predict(x_test1)\n", + "rf_model1 = RandomForestClassifier()\n", + "rf_model1.fit(x_train1, y_train)\n", + "rf_pred1 = rf_model1.predict(x_test1)\n", "acc1 = accuracy_score(y_test, rf_pred1) * 100\n", "accuracies1.append(acc1)\n", "# format with color coded if acc1 > 90\n", "acc1 = f\"\\033[92m{acc1:.2f}\\033[00m\" if acc1 > 90 else f\"{acc1:.2f}\"\n", "print(\"Random Forest Accuracy for sensor 1:\", acc1)\n", - "rf_model.fit(x_train2, y_train)\n", - "rf_pred2 = rf_model.predict(x_test2)\n", + "rf_model2 = RandomForestClassifier()\n", + "rf_model2.fit(x_train2, y_train)\n", + "rf_pred2 = rf_model2.predict(x_test2)\n", "acc2 = accuracy_score(y_test, rf_pred2) * 100\n", "accuracies2.append(acc2)\n", "# format with color coded if acc2 > 90\n", @@ -616,16 +644,17 @@ "# print(y_test)\n", "\n", "# 2. Bagged Trees\n", - "bagged_model = BaggingClassifier(estimator=DecisionTreeClassifier(), n_estimators=10)\n", - "bagged_model.fit(x_train1, y_train)\n", - "bagged_pred1 = bagged_model.predict(x_test1)\n", + "bagged_model1 = BaggingClassifier(estimator=DecisionTreeClassifier(), n_estimators=10)\n", + "bagged_model1.fit(x_train1, y_train)\n", + "bagged_pred1 = bagged_model1.predict(x_test1)\n", "acc1 = accuracy_score(y_test, bagged_pred1) * 100\n", "accuracies1.append(acc1)\n", "# format with color coded if acc1 > 90\n", "acc1 = f\"\\033[92m{acc1:.2f}\\033[00m\" if acc1 > 90 else f\"{acc1:.2f}\"\n", "print(\"Bagged Trees Accuracy for sensor 1:\", acc1)\n", - "bagged_model.fit(x_train2, y_train)\n", - "bagged_pred2 = bagged_model.predict(x_test2)\n", + "bagged_model2 = BaggingClassifier(estimator=DecisionTreeClassifier(), n_estimators=10)\n", + "bagged_model2.fit(x_train2, y_train)\n", + "bagged_pred2 = bagged_model2.predict(x_test2)\n", "acc2 = accuracy_score(y_test, bagged_pred2) * 100\n", "accuracies2.append(acc2)\n", "# format with color coded if acc2 > 90\n", @@ -641,8 +670,9 @@ "# format with color coded if acc1 > 90\n", "acc1 = f\"\\033[92m{acc1:.2f}\\033[00m\" if acc1 > 90 else f\"{acc1:.2f}\"\n", "print(\"Decision Tree Accuracy for sensor 1:\", acc1)\n", - "dt_model.fit(x_train2, y_train)\n", - "dt_pred2 = dt_model.predict(x_test2)\n", + "dt_model2 = DecisionTreeClassifier()\n", + "dt_model2.fit(x_train2, y_train)\n", + "dt_pred2 = dt_model2.predict(x_test2)\n", "acc2 = accuracy_score(y_test, dt_pred2) * 100\n", "accuracies2.append(acc2)\n", "# format with color coded if acc2 > 90\n", @@ -658,8 +688,9 @@ "# format with color coded if acc1 > 90\n", "acc1 = f\"\\033[92m{acc1:.2f}\\033[00m\" if acc1 > 90 else f\"{acc1:.2f}\"\n", "print(\"KNeighbors Accuracy for sensor 1:\", acc1)\n", - "knn_model.fit(x_train2, y_train)\n", - "knn_pred2 = knn_model.predict(x_test2)\n", + "knn_model2 = KNeighborsClassifier()\n", + "knn_model2.fit(x_train2, y_train)\n", + "knn_pred2 = knn_model2.predict(x_test2)\n", "acc2 = accuracy_score(y_test, knn_pred2) * 100\n", "accuracies2.append(acc2)\n", "# format with color coded if acc2 > 90\n", @@ -675,8 +706,9 @@ "# format with color coded if acc1 > 90\n", "acc1 = f\"\\033[92m{acc1:.2f}\\033[00m\" if acc1 > 90 else f\"{acc1:.2f}\"\n", "print(\"Linear Discriminant Analysis Accuracy for sensor 1:\", acc1)\n", - "lda_model.fit(x_train2, y_train)\n", - "lda_pred2 = lda_model.predict(x_test2)\n", + "lda_model2 = LinearDiscriminantAnalysis()\n", + "lda_model2.fit(x_train2, y_train)\n", + "lda_pred2 = lda_model2.predict(x_test2)\n", "acc2 = accuracy_score(y_test, lda_pred2) * 100\n", "accuracies2.append(acc2)\n", "# format with color coded if acc2 > 90\n", @@ -692,8 +724,9 @@ "# format with color coded if acc1 > 90\n", "acc1 = f\"\\033[92m{acc1:.2f}\\033[00m\" if acc1 > 90 else f\"{acc1:.2f}\"\n", "print(\"Support Vector Machine Accuracy for sensor 1:\", acc1)\n", - "svm_model.fit(x_train2, y_train)\n", - "svm_pred2 = svm_model.predict(x_test2)\n", + "svm_model2 = SVC()\n", + "svm_model2.fit(x_train2, y_train)\n", + "svm_pred2 = svm_model2.predict(x_test2)\n", "acc2 = accuracy_score(y_test, svm_pred2) * 100\n", "accuracies2.append(acc2)\n", "# format with color coded if acc2 > 90\n", @@ -709,8 +742,9 @@ "# format with color coded if acc1 > 90\n", "acc1 = f\"\\033[92m{acc1:.2f}\\033[00m\" if acc1 > 90 else f\"{acc1:.2f}\"\n", "print(\"XGBoost Accuracy:\", acc1)\n", - "xgboost_model.fit(x_train2, y_train)\n", - "xgboost_pred2 = xgboost_model.predict(x_test2)\n", + "xgboost_model2 = XGBClassifier()\n", + "xgboost_model2.fit(x_train2, y_train)\n", + "xgboost_pred2 = xgboost_model2.predict(x_test2)\n", "acc2 = accuracy_score(y_test, xgboost_pred2) * 100\n", "accuracies2.append(acc2)\n", "# format with color coded if acc2 > 90\n", @@ -804,7 +838,6 @@ "\n", " # df1['s1'] = sensor1[sensor1.columns[-1]]\n", " # df1['s2'] = sensor2[sensor2.columns[-1]]\n", - "ed\n", " # # Combined Plot for sensor 1 and sensor 2 from data1 file in which motor is operated at 800 rpm\n", "\n", " # plt.plot(df1['s2'], label='sensor 2')\n", @@ -840,7 +873,49 @@ "metadata": {}, "outputs": [], "source": [ - "spectograph('D:/thesis/data/converted/raw')" + "from src.ml.model_selection import create_ready_data\n", + "\n", + "X1b, y = create_ready_data('D:/thesis/data/converted/raw_B/sensor1')\n", + "X2b, y = create_ready_data('D:/thesis/data/converted/raw_B/sensor2')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "y.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score, classification_report\n", + "# 4. Validate on Dataset B\n", + "y_pred = svm_model.predict(X1b)\n", + "\n", + "# 5. Evaluate\n", + "print(\"Accuracy on Dataset B:\", accuracy_score(y, y_pred))\n", + "print(classification_report(y, y_pred))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score, classification_report\n", + "# 4. Validate on Dataset B\n", + "y_pred = svm_model2.predict(X2b)\n", + "\n", + "# 5. Evaluate\n", + "print(\"Accuracy on Dataset B:\", accuracy_score(y, y_pred))\n", + "print(classification_report(y, y_pred))" ] } ],