Chapter 13 - I have a problem trying to implement the autograd in a simple linear regresion

[HectorPulido]

I tried this, mi input size is 1, 1000 and my output size is 1, 100

import random as r

x = np.array(range(1000))
y = x * 12 + 15
y = y + np.random.randn(*y.shape)

x = x.reshape(-1 , 1)
y = y.reshape(-1 , 1)

data = Tensor(x, autograd=True)
target = Tensor(y, autograd=True)

w = list()
w.append(Tensor(np.random.rand(1,1), autograd=True))
w.append(Tensor(np.random.rand(1), autograd=True))
for i in range(10):

  pred = data.mm(w[0]) + w[1].expand(0,1000)
  loss = ((pred - target)*(pred - target)).sum(0)

  loss.backward(Tensor(np.ones_like(loss.data)))
  for w_ in w:
    w_.data -= w_.grad.data * 0.1
    w_.grad.data *= 0
  print(loss)

OUTPUT

[4.20028134e+10]
[1.86120338e+26]
[8.24726275e+41]
[3.65448202e+57]
[1.61935411e+73]
[7.17559347e+88]
[3.17960978e+104]
[1.40893132e+120]
[6.2431795e+135]
[2.7664436e+151]

[HectorPulido]

10000 years later, this is the fixed code

from Autograd import Tensor
import numpy as np
import random as r

np.random.seed(0)

# True weights and bias for the linear equation
true_w = 15
true_b = 17

# Define data points and labels 
x_values = np.linspace(-10, 10, 100)
y_values = true_w * x_values + true_b

# Introduce noise in the labels 
y_values += 0.1 * np.random.randn(*x_values.shape)

# Convert numpy arrays to Tensor objects
X = Tensor(x_values, autograd=True)
y = Tensor(y_values, autograd=True)

# Initialize random weights and bias
w = Tensor(np.random.rand(1), autograd=True)
b = Tensor(np.random.rand(1), autograd=True)

# Set learning rate
learning_rate = 0.01

# Training loop
for i in range(10000):
    # forward pass
    preds = X * w + b
    loss = ((preds - y) * (preds - y)).sum(0)

    # backward pass
    loss.backward(Tensor(np.ones_like(loss.data)))

    # applying the gradient descent step
    w.data -= learning_rate * np.mean(w.grad.data)
    b.data -= learning_rate * np.mean(b.grad.data)

    # manually zero gradients after updating weights
    w.grad.data *= 0
    b.grad.data *= 0 

    # print the loss
    if i % 10 == 0:
        print(f"Loss at step {i}: {loss.data}")

print(f"True values: w = {true_w}, b = {true_b}")
print(f"Trained values: w = {w.data[0]}, b = {b.data[0]}")

Loss at step 0: 7945.279089364454
Loss at step 10: 34.13420435735701
Loss at step 20: 23.123420765656913
Loss at step 30: 15.772534527085597
Loss at step 40: 10.865024274750603
Loss at step 50: 7.588731308821935
Loss at step 60: 5.401452006962766
...
Loss at step 9970: 1.008310853468597
Loss at step 9980: 1.008310853468597
Loss at step 9990: 1.008310853468597
True values: w = 15, b = 17
Trained values: w = 14.998513365676923, b = 17.00598080155336