Code not working. Here is a fix:

[serlou]

Universal Function Approximation from http://deliprao.com/archives/100

by Delip on November 13, 2015 in Deep Learning, Machine Learning

A multilayered neural network with even a single hidden layer can learn any function. This universal function approximation property of multilayer perceptrons was first noted by Cybenko (1989) and Hornik (1991). In this post, I will use TensorFlow to implement a multilayer neural network (also known as a multilayer perceptron) to learn arbitrary Python lambda expressions.

Translated into PyTorch from TensorFlow
%matplotlib inline

import numpy as np
import math, random
import matplotlib.pyplot as plt

np.random.seed(1000) # for repro

NUM_HIDDEN_NODES = 20
NUM_EXAMPLES = 1000
TRAIN_SPLIT = .8
NUM_EPOCHS = 5000
function_to_learn = lambda x: np.sin(x) + 0.1np.random.randn(x.shape)
all_x = np.float32(np.random.uniform(-2math.pi, 2math.pi, (1, NUM_EXAMPLES))).T

np.random.shuffle(all_x)

train_size = int(NUM_EXAMPLES*TRAIN_SPLIT)

trainx = all_x[:train_size]
trainy = function_to_learn(trainx)

testx = all_x[train_size:]
testy = function_to_learn(testx)
plt.figure(1)
plt.scatter(trainx, trainy, c='green', label='train')
plt.scatter(testx, testy, c='red', label='validation')
plt.legend()

N is batch size; D_in is input dimension;

H is hidden dimension; D_out is output dimension.

from (https://github.com/jcjohnson/pytorch-examples)

N, D_in, H, D_out = NUM_EXAMPLES, 1, NUM_HIDDEN_NODES, 1
import torch
from torch.autograd import Variable
import torch.nn.init as init
import torch.nn.functional as F

dtype = torch.FloatTensor
class Net(torch.nn.Module):
def init(self, D_in, H, D_out):
super(Net, self).init()
self.w1 = torch.nn.Linear(D_in, H)
self.w2 = torch.nn.Linear(H, D_out)

def forward(self, x):
    h_relu = F.tanh(self.w1(x).clamp(min=0))
    y_pred = self.w2(h_relu)
    return y_pred

x = Variable(torch.FloatTensor(trainx))
y = Variable(torch.FloatTensor(trainy))
model = Net(D_in, H, D_out)

glorot_weight_zero_bias(model)

criterion = torch.nn.MSELoss(size_average=False)
optimizer = torch.optim.SGD(model.parameters(), lr=1e-5)

for t in range(NUM_EPOCHS):

optimizer.zero_grad()

y_pred = model(x)
loss = criterion(y_pred, y)
loss.backward()
optimizer.step()

print(t, loss.item())
x = Variable(torch.FloatTensor(testx))
y = Variable(torch.FloatTensor(testy))
y_pred = model(x)
plt.figure(1)
plt.scatter(testx, testy, c='red', label='validation')
plt.scatter(testx, y_pred.data.numpy(), c='green', label='test')
plt.legend()