feat: Add absolute value option to time feature extraction

Introduce a Python script for transforming QUGS 2D grid structure data into a simplified 1D beam format suitable for SVM-based damage detection. The script efficiently slices original CSV files into smaller, manageable sets, correlating specific damage scenarios with their corresponding sensor data. This change addresses the challenge of retaining critical damage localization information during the data conversion process, ensuring high-quality, relevant data for 1D analysis.

Closes #20

.gitignore

@@ -1,4 +1,4 @@
 # Ignore CSV files in the data directory and all its subdirectories
 data/**/*.csv
-
+.venv/
 *.pyc

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Python Debugger: Current File with Arguments",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal",
+      "args": ["data/raw", "data/raw"]
+    }
+  ]
+}

code/src/features/build_features.py

@@ -1,16 +1,39 @@
 # src/features/build_features.py
 import pandas as pd
-from time_domain_features import FeatureExtractor
-import numpy as np
+from time_domain_features import ExtractTimeFeatures
+import os
+import re
 
-def build_features(input_file, output_file):
-    data = pd.read_csv(input_file)
-    # Assuming the relevant data is in the first column
-    extractor = FeatureExtractor(data.iloc[:, 0].values)
-    features = extractor.features
+# define function, regex pattern for extracting the damage level and test number store in pairs array
+def extract_numbers(filename):
+    # Find all occurrences of one or more digits in the filename
+    numbers = re.findall(r'\d+', filename)
+    # Convert the list of number strings to integers
+    numbers = [int(num) for num in numbers]
+    # Convert to a tuple and return
+    return print(tuple(numbers))
 
+def build_features(input_dir, output_dir):
+    all_features = []
+    for nth_damage in os.listdir(input_dir):
+        nth_damage_path = os.path.join(input_dir, nth_damage)
+        if os.path.isdir(nth_damage_path):
+            print(nth_damage)
+            for nth_test in os.listdir(nth_damage_path):
+                nth_test_path = os.path.join(nth_damage_path, nth_test)
+                # print(nth_test_path)
+                features = ExtractTimeFeatures(nth_test_path) # return the one csv file feature in dictionary {}
+                all_features.append(features)
+
+    # Create a DataFrame from the list of dictionaries
+    df = pd.DataFrame(all_features)
+    print(df)
+    # Save the DataFrame to a CSV file in the output directory
+    output_file_path = os.path.join(output_dir, 'combined_features.csv')
+    df.to_csv(output_file_path, index=False)
+    print(f"Features saved to {output_file_path}")
     # Save features to a file
-    np.savez(output_file, **features)
+    # np.savez(output_file, **features)
 
 if __name__ == "__main__":
     import sys
@@ -18,4 +41,4 @@ if __name__ == "__main__":
     output_path = sys.argv[2]  # 'data/features/feature_matrix.npz'
     
     # Assuming only one file for simplicity; adapt as needed
-    build_features(f"{input_path}processed_data.csv", output_path)
+    build_features(input_path, output_path)

code/src/features/time_domain_features.py

@@ -36,6 +36,16 @@ class FeatureExtractor:
             result += f"{feature}: {value:.4f}\n"
         return result
 
+def ExtractTimeFeatures(object, absolute):
+    data = pd.read_csv(object, skiprows=1) # Skip the header row separator char info
+    if absolute:
+        extractor = FeatureExtractor(np.abs(data.iloc[:, 1].values)) # Assuming the data is in the second column
+    else:
+        extractor = FeatureExtractor(data.iloc[:, 1].values)
+    features = extractor.features
+    return features
+    # Save features to a file
+    # np.savez(output_file, **features)
 # Usage
 # Assume you have a CSV file with numerical data in the first column
 # Create an instance of the class and pass the path to your CSV file

data/QUGS/convert.py

@@ -0,0 +1,65 @@
+import pandas as pd
+import os
+import sys
+from colorama import Fore, Style, init
+
+def create_damage_files(base_path, output_base):
+    # Initialize colorama
+    init(autoreset=True)
+    
+    # Generate column labels based on expected duplication in input files
+    columns = ['Real'] + [f'Real.{i}' for i in range(1, 30)]  # Explicitly setting column names
+
+    sensor_end_map = {1: 'Real.25', 2: 'Real.26', 3: 'Real.27', 4: 'Real.28', 5: 'Real.29'}
+
+    # Define the damage scenarios and the corresponding original file indices
+    damage_scenarios = {
+        1: range(6, 11),  # Damage 1 files from zzzAD6.csv to zzzAD10.csv
+        2: range(11, 16), # Damage 2 files from zzzAD11.csv to zzzAD15.csvs
+        3: range(16, 21), # Damage 3 files from zzzAD16.csv to zzzAD20.csv
+        4: range(21, 26)  # Damage 4 files from zzzAD21.csv to zzzAD25.csv
+    }
+    damage_pad = len(str(len(damage_scenarios)))
+    test_pad = len(str(30))
+
+    for damage, files in damage_scenarios.items():
+        for i, file_index in enumerate(files, start=1):
+            # Load original data file
+            file_path = os.path.join(base_path, f'zzzAD{file_index}.TXT')
+            df = pd.read_csv(file_path, sep='\t', skiprows=10)  # Read with explicit column names
+
+            top_sensor = columns[i-1]
+            print(top_sensor, type(top_sensor))
+            output_file_1 = os.path.join(output_base, f'DAMAGE_{damage}', f'D{damage:0{damage_pad}}_TEST{i:0{test_pad}}_01.csv')
+            print(f"Creating {output_file_1} from taking zzzAD{file_index}.TXT")
+            print("Taking datetime column on index 0...")
+            print(f"Taking `{top_sensor}`...")
+            df[['Time', top_sensor]].to_csv(output_file_1, index=False)
+            print(Fore.GREEN + "Done")
+
+            bottom_sensor = sensor_end_map[i]
+            output_file_2 = os.path.join(output_base, f'DAMAGE_{damage}', f'D{damage}_TEST{i}_02.csv')
+            print(f"Creating {output_file_2} from taking zzzAD{file_index}.TXT")
+            print("Taking datetime column on index 0...")
+            print(f"Taking `{bottom_sensor}`...")
+            df[['Time', bottom_sensor]].to_csv(output_file_2, index=False)
+            print(Fore.GREEN + "Done")
+            print("---")
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python convert.py <path_to_csv_files>")
+        sys.exit(1)
+
+    base_path = sys.argv[1]
+    output_base = sys.argv[2]  # Define output directory
+
+    # Create output folders if they don't exist
+    for i in range(1, 5):
+        os.makedirs(os.path.join(output_base, f'DAMAGE_{i}'), exist_ok=True)
+
+    create_damage_files(base_path, output_base)
+    print(Fore.YELLOW + Style.BRIGHT + "All files have been created successfully.")
+
+if __name__ == "__main__":
+    main()

generate_dummy_data.py

@@ -16,8 +16,10 @@ os.makedirs(processed_path, exist_ok=True)
 # Define the number of zeros to pad
 num_damages = 5
 num_tests = 10
+num_sensors = 2
 damage_pad = len(str(num_damages))
 test_pad = len(str(num_tests))
+sensor_pad = len(str(num_sensors))
 
 for damage in range(1, num_damages + 1):  # 5 Damage levels starts from 1
     damage_folder = f"DAMAGE_{damage:0{damage_pad}}"
@@ -25,8 +27,9 @@ for damage in range(1, num_damages + 1):  # 5 Damage levels starts from 1
     os.makedirs(damage_path, exist_ok=True)
 
     for test in range(1, 11):  # 10 Tests per damage level
+        for sensor in range(1, 3):  # 2 Sensors per test
             # Filename for the CSV
-        csv_filename = f"D{damage:0{damage_pad}}_TEST{test:0{test_pad}}.csv"
+            csv_filename = f"D{damage:0{damage_pad}}_TEST{test:0{test_pad}}_{sensor:0{sensor_pad}}.csv"
             csv_path = os.path.join(damage_path, csv_filename)
             
             # Generate dummy data