USER
i have following code to train my NN model :
# %%
from sklearn.preprocessing import StandardScaler
import pandas as pd
import numpy as np
from tensorflow import keras
import joblib
def calculate_targets_scaling_params(file_path):
scaler = StandardScaler()
for chunk in pd.read_csv(file_path, chunksize=10000): # Adjust chunksize based on your memory capacity
filtered_chunk = chunk[['y_High_5d','y_Low_5d','y_Priority_5d']]
# 'y_Priority_1d',
# 'y_High_2d', 'y_Low_2d', 'y_Priority_2d',
# 'y_High_3d', 'y_Low_3d', 'y_Priority_3d',
# 'y_High_5d', 'y_Low_5d', 'y_Priority_5d'
scaler.partial_fit(filtered_chunk) # Accumulate means and variances
return scaler.mean_, scaler.var_
# %%
x_scaler_loaded = joblib.load('nn_x_scaler.sav')
y_scaler_loaded = joblib.load('nn_y_hlp5_scaler.sav')
def data_generator(file_path, batch_size, x_scaler, y_scaler):
chunksize = batch_size
while True: # Loop forever, so the generator never terminates
for chunk in pd.read_csv(file_path, chunksize=chunksize):
# if data_type == 'train' and row_counter >= train_rows:
# continue # Skip the rest if we are fetching training data but have reached the end of the train set
# elif data_type == 'val' and row_counter < train_rows:
# row_counter += len(chunk)
# continue # Skip this chunk if we are fetching validation data but are still in the train range
# Assuming your CSV has headers that match features/targets
# Normalizing the features
filtered_c = chunk.drop(['Date', 'Symbol'], axis=1)
feature_data = filtered_c.drop([
'y_High_1d', 'y_Low_1d', 'y_Priority_1d',
'y_High_2d', 'y_Low_2d', 'y_Priority_2d',
'y_High_3d', 'y_Low_3d', 'y_Priority_3d',
'y_High_5d', 'y_Low_5d', 'y_Priority_5d'], axis=1)
target_data = filtered_c[['y_High_5d', 'y_Low_5d', 'y_Priority_5d'
# , 'y_Low_1d', 'y_Priority_1d',
# 'y_High_2d', 'y_Low_2d', 'y_Priority_2d',
# 'y_High_3d', 'y_Low_3d', 'y_Priority_3d',
# 'y_High_5d', 'y_Low_5d', 'y_Priority_5d'
]]
feature_data_scaled = pd.DataFrame(x_scaler.transform(feature_data), columns=feature_data.columns)
# Assuming target_data also needs to be scaled, apply scaler separately
target_data_scaled = pd.DataFrame(y_scaler.transform(target_data), columns=target_data.columns)
# Now, feature_data_scaled and target_data_scaled are both DataFrames, scaled and ready to use
yield feature_data_scaled.values, target_data_scaled.values
# row_counter += len(chunk)
# %%
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Input
import tensorflow as tf
from tensorflow.keras.layers import BatchNormalization
def build_model():
input_shape = (6427,)
model = Sequential([
Dense(6427, activation='relu', input_shape = input_shape),
Dropout(0.17),
BatchNormalization(),
Dense(4096, activation='relu'),
Dropout(0.15),
BatchNormalization(),
Dense(2048, activation='relu'),
Dropout(0.12),
BatchNormalization(),
Dense(1024, activation='relu'),
Dropout(0.10),
BatchNormalization(),
Dense(512, activation='relu'),
Dropout(0.05),
BatchNormalization(),
Dense(256, activation='relu'),
BatchNormalization(),
Dense(128, activation='relu'),
BatchNormalization(),
Dense(64, activation='relu'),
BatchNormalization(),
Dense(32, activation='relu'),
BatchNormalization(),
Dense(3),
])
optimizer = keras.optimizers.Adam(learning_rate=0.006)
model.compile(optimizer='adam',
loss='mse', # Use Mean Squared Error for regression
metrics=['mae']) # Mean Absolute Error as an additional metric
return model
# %%
# Instantiate the model
model = build_model()
model.summary()
# %%
import warnings
warnings.filterwarnings(action='ignore', message='X has feature names, but StandardScaler was fitted without feature names')
train_generator = data_generator(file_path, batch_size,x_scaler=x_scaler_loaded,y_scaler=y_scaler_loaded)
total_samples = 301617 # Assuming same example size
train_samples = int(total_samples)
steps_per_epoch = train_samples // batch_size
# Modify the model fitting call to include validation data
model.fit(
train_generator,
steps_per_epoch=steps_per_epoch,
epochs=45,
)
currently my model not using any validation dataset when training
change the code properly for implementing val setASSISTANT
To implement a validation set in your training process, you will need to modify your `data_generator` function to distinguish between training and validation data, and then create a separate generator for validation data. Additionally, you will need to determine the size of your validation set to calculate the `validation_steps` for the `.fit()` call.
Here is a modified version of your code that includes these changes:
### Step 1: Modify `data_generator` to Accept `data_type` Argument
Modify the `data_generator` function so it can differentiate between training and validation data based on a new `data_type` argument. This requires adjusting your data loading logic to split your dataset accordingly.
```python
def data_generator(file_path, batch_size, x_scaler, y_scaler, data_type='train', validation_split=0.2):
# Total number of rows in the file
total_rows = sum(1 for _ in open(file_path)) - 1 # Subtract 1 for the header
train_rows = int(total_rows * (1 - validation_split))
row_counter = 0
chunksize = batch_size
while True: # Loop forever
for chunk in pd.read_csv(file_path, chunksize=chunksize):
# Determine whether to process this chunk based on data_type and row_counter
if data_type == 'train' and row_counter >= train_rows:
row_counter += len(chunk) # Increment row_counter but do not use chunk
continue
elif data_type == 'val' and row_counter < train_rows:
row_counter += len(chunk) # Increment row_counter but do not use chunk
continue
# Your existing processing code here, such as filtering and scaling
if data_type == 'val' and row_counter >= total_rows:
row_counter = 0 # Reset for validation generator loop
yield feature_data_scaled.values, target_data_scaled.values
row_counter += len(chunk)
```
### Step 2: Create Separate Generators for Training and Validation
Instantiate separate generators for both training and validation data using the modified `data_generator` function.
```python
batch_size = 128 # Define your batch size
file_path = 'your_file_path.csv' # Define your file path
# Training generator
train_generator = data_generator(file_path, batch_size, x_scaler=x_scaler_loaded, y_scaler=y_scaler_loaded, data_type='train')
# Validation generator
val_generator = data_generator(file_path, batch_size, x_scaler=x_scaler_loaded, y_scaler=y_scaler_loaded, data_type='val', validation_split=0.2)
```
### Step 3: Update `.fit()` Call to Include Validation Data
Calculate `validation_steps` and update the `model.fit()` call.
```python
total_samples = 301617 # Total example size
validation_samples = total_samples * 0.2 # Assuming 20% of the data is for validation
steps_per_epoch = int((total_samples * 0.8) // batch_size) # Training steps
validation_steps = int(validation_samples // batch_size) # Validation steps
model.fit(
train_generator,
steps_per_epoch=steps_per_epoch,
validation_data=val_generator,
validation_steps=validation_steps,
epochs=45,
)
```
### Note:
- Adjust `validation_split` in the `data_generator` and `validation_samples` calculation if you decide on a different size for your validation set.
- Ensure the `row_counter` reset logic aligns with your specific needs, especially in the edge cases at the start/end of your data.
- The example assumes that `training` and `validation` data are sequentially organized. If your dataset is not in this format, consider shuffling or pre-organizing your dataset accordingly.