feat(data-processing): Implement CSV data transformation for SVM analysis

Introduce a Python script for transforming QUGS 2D grid structure data into a simplified 1D beam format suitable for SVM-based damage detection. The script efficiently slices original CSV files into smaller, manageable sets, correlating specific damage scenarios with their corresponding sensor data. This change addresses the challenge of retaining critical damage localization information during the data conversion process, ensuring high-quality, relevant data for 1D analysis.

Closes #20
This commit is contained in:
nuluh
2024-09-03 11:33:23 +07:00
parent 0306f28a68
commit d12eea0acf

63
data/QUGS/convert.py Normal file
View File

@@ -0,0 +1,63 @@
import pandas as pd
import os
import sys
from colorama import Fore, Style, init
def create_damage_files(base_path, output_base):
# Initialize colorama
init(autoreset=True)
# Generate column labels based on expected duplication in input files
columns = ['Real'] + [f'Real.{i}' for i in range(1, 30)] # Explicitly setting column names
sensor_end_map = {1: 'Real.25', 2: 'Real.26', 3: 'Real.27', 4: 'Real.28', 5: 'Real.29'}
# Define the damage scenarios and the corresponding original file indices
damage_scenarios = {
1: range(6, 11), # Damage 1 files from zzzAD6.csv to zzzAD10.csv
2: range(11, 16), # Damage 2 files from zzzAD11.csv to zzzAD15.csv
3: range(16, 21), # Damage 3 files from zzzAD16.csv to zzzAD20.csv
4: range(21, 26) # Damage 4 files from zzzAD21.csv to zzzAD25.csv
}
for damage, files in damage_scenarios.items():
for i, file_index in enumerate(files, start=1):
# Load original data file
file_path = os.path.join(base_path, f'zzzAD{file_index}.TXT')
df = pd.read_csv(file_path, sep='\t', skiprows=10) # Read with explicit column names
top_sensor = columns[damage-1]
print(top_sensor, type(top_sensor))
output_file_1 = os.path.join(output_base, f'DAMAGE_{damage}', f'D{damage}_TEST{i}_01.csv')
print(f"Creating {output_file_1} from taking zzzAD{file_index}.TXT")
print("Taking datetime column on index 0...")
print(f"Taking `{top_sensor}`...")
df[['Time', top_sensor]].to_csv(output_file_1, index=False)
print(Fore.GREEN + "Done")
bottom_sensor = sensor_end_map[damage]
output_file_2 = os.path.join(output_base, f'DAMAGE_{damage}', f'D{damage}_TEST{i}_02.csv')
print(f"Creating {output_file_2} from taking zzzAD{file_index}.TXT")
print("Taking datetime column on index 0...")
print(f"Taking `{bottom_sensor}`...")
df[['Time', bottom_sensor]].to_csv(output_file_2, index=False)
print(Fore.GREEN + "Done")
print("---")
def main():
if len(sys.argv) < 2:
print("Usage: python convert.py <path_to_csv_files>")
sys.exit(1)
base_path = sys.argv[1]
output_base = "output_data" # Define output directory
# Create output folders if they don't exist
for i in range(1, 5):
os.makedirs(os.path.join(output_base, f'DAMAGE_{i}'), exist_ok=True)
create_damage_files(base_path, output_base)
print(Fore.YELLOW + Style.BRIGHT + "All files have been created successfully.")
if __name__ == "__main__":
main()