Performance issues with regression models

[chn7960]

Hi, I am developing a regression model using snnTorch.
As a dataset, I'm using sklearn's california housing dataset.
I wrote the code while learning the regression tutorial, and it works.
However, the model performance is very poor.
What's the matter? Can we discuss this issue together?
I share my code below.

---

import snntorch as snn
from snntorch import surrogate
from snntorch import functional as SF
from snntorch import utils
import torch
import torch.nn as nn
from torch.utils.data import random_split, DataLoader, TensorDataset 
import torch.nn.functional as F
from sklearn.datasets import fetch_california_housing
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt
import numpy as np
import itertools
import tqdm
import pandas as pd
from sklearn.metrics import mean_squared_error, r2_score     

dtype = torch.float
device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")

housing = fetch_california_housing(as_frame=True) 
X = housing.data
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
X_scaled = torch.tensor(X_scaled, dtype=dtype)
y = housing.target*10 # rescaling
y = torch.tensor(y, dtype=int)
ds = TensorDataset(X_scaled, y)
number_rows = len(ds)
test_split = int(number_rows*0.3)
print(test_split)
train_split = number_rows - test_split
housing_train, housing_test = random_split(ds, [train_split, test_split])    
batch_size = 100
train_loader = DataLoader(housing_train, batch_size=batch_size, shuffle=True, drop_last=True)
test_loader = DataLoader(housing_test, batch_size=batch_size, shuffle=True, drop_last=True)

num_inputs = X_scaled.size(1)
num_hidden = 1000
num_outputs = (max(y)+1)
num_steps = 20
beta = 0.9

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.fc1 = nn.Linear(num_inputs, num_hidden)
        self.rlif1 = snn.RLeaky(beta, linear_features=num_hidden) #linear_features=num_hidden
        self.fc2 = nn.Linear(num_hidden, num_outputs)
        self.rlif2 = snn.RLeaky(beta, linear_features=num_outputs) #linear_features=num_outputs

    def forward(self, x):
        spk1, mem1 = self.rlif1.init_rleaky()
        spk2, mem2 = self.rlif2.init_rleaky()
        spk2_rec = []
        for step in range(num_steps):
            cur1 = self.fc1(x)
            spk1, mem1 = self.rlif1(cur1, spk1, mem1)
            cur2 = self.fc2(spk1)
            spk2, mem2 = self.rlif2(cur2, spk2, mem2)
            spk2_rec.append(spk2)
        return torch.stack(spk2_rec)
        
net = Net().to(device)

def print_batch_accuracy(data, targets, train=False):
    spk = net(data.flatten(1))
    total = targets.size(0)
    correct = SF.accuracy_rate(spk, targets) * spk.size(1)    

    if train:
        print(f"Train set accuracy for a single minibatch: {correct / total*100:.2f}%")
    else:
        print(f"Test set accuracy for a single minibatch: {correct / total*100:.2f}%")

def train_printer(
    data, targets, epoch,
    counter, iter_counter,
        loss_hist, test_loss_hist, test_data, test_targets):
    print(f"Epoch {epoch}, Iteration {iter_counter}")
    print(f"Train Set Loss: {loss_hist[counter]:.2f}")
    print(f"Test Set Loss: {test_loss_hist[counter]:.2f}")
    print_batch_accuracy(data, targets, train=True)
    print_batch_accuracy(test_data, test_targets, train=False)
    print("\n")

loss = SF.mse_count_loss(correct_rate=0.8, incorrect_rate=0.2)
optimizer = torch.optim.Adam(net.parameters(), lr=1e-3)
num_epochs = 10
loss_hist = []
test_loss_hist = []
counter = 0

for epoch in range(num_epochs):
    iter_counter = 0
    train_batch = iter(train_loader)
      
    for data, targets in train_batch:
        data = data.to(device)
        targets = targets.to(device)     
        spk = net(data.flatten(1))    
        loss_val = torch.zeros((1), dtype=dtype, device=device)
        for step in range(num_steps):
            loss_val += loss(spk, targets)
                        
        optimizer.zero_grad()
        loss_val.backward()
        optimizer.step()
        
        loss_hist.append(loss_val.item()) 

        with torch.no_grad():
            net.eval()
            test_data, test_targets = next(iter(test_loader))
            test_data = test_data.to(device)
            test_targets = test_targets.to(device)
            spk = net(test_data.flatten(1))
            test_loss = torch.zeros((1), dtype=dtype, device=device)
            for step in range(num_steps):
                test_loss += loss(spk, test_targets)         
            test_loss_hist.append(test_loss.item())
            
            if counter % 10 == 0:
                train_printer(
                    data, targets, epoch,
                    counter, iter_counter,
                    loss_hist, test_loss_hist,
                    test_data, test_targets)
            counter += 1
            iter_counter +=1

total = 0
correct = 0
predictions = torch.tensor([], dtype=torch.float)
actual = torch.tensor([], dtype=torch.float)

with torch.no_grad():
    net.eval()
    for data, targets in test_loader:
        data = data.to(device)
        targets = targets.to(device)
        spk = net(data.flatten(1))

        _, idx = spk.sum(dim=0).max(1)
        total += targets.size(0)
        correct += SF.accuracy_rate(spk, targets) * spk.size(1)
        predictions = torch.cat((predictions, idx), 0)
        actual = torch.cat((actual, targets), 0)
        
predictions = predictions.numpy()
actual = actual.numpy()
mse = mean_squared_error(predictions, actual)
rmse = np.sqrt(mse)
r2 = r2_score(predictions, actual)

print(f"Total correctly classified test set images: {correct}/{total}")
print(f"Test Set Accuracy: {100 * correct / total:.2f}%")
print(f"Test Set MSE: {mse:.2f}")
print(f"Test Set RMSE: {rmse:.2f}")
print(f"Test Set r-squared: {r2:.2f}")

[jeshraghian]

I don't think you're using a loss function that is representative of your task:

loss = SF.mse_count_loss(correct_rate=0.8, incorrect_rate=0.2)

The purpose of mse_count_loss in the way that you have set it up is as a classification task. The correct class fires 80% of the time, and the incorrect fires 20% of the time.

[chn7960]

Thank you for answer.
So which loss function should I use?
Should I use the L1_loss or mse_membrane_loss?

[Whale-xh]

Hello, I also encountered the same problem, do you have any solution?

[Whale-xh]

Hello, I also encountered the same problem, have you solved it?