grad.py

import torch
import numpy as np
import matplotlib.pyplot as plt

# Imagine this is loss function 
def polynomial(w):
    return (w**5 - 3*w**4 + 2*w**3 + w**2 - w + 1)/100000

# Imagine this is loss function multiplied by 10000
def scaled_polynomial(w):
    return (w**5 - 3*w**4 + 2*w**3 + w**2 - w + 1)

# Function to perform gradient descent
def gradient_descent(func, λ, num_iterations, initial_x):
    x = torch.tensor(initial_x, requires_grad=True)
    x_values = []
    for i in range(num_iterations):
        value = func(x)
        value.backward()
        with torch.no_grad():
            x -= λ * x.grad
        x.grad.zero_()
        x_values.append(x.item())
    return x_values

# Same number of iterations for both cases
num_iterations = 1000

# Case 1
λ = 100
x_min_values = gradient_descent(polynomial, λ, num_iterations, 1.0)

# Case 2
λ = 0.01
x_min_scaled_values = gradient_descent(scaled_polynomial, λ, num_iterations, 1.0)

# Compare the sequence of x values
print("Sequence of w values are the same:", np.array_equal(x_min_values, x_min_scaled_values))