Analysing a NN Problem (Randomly Sampled)

Simulate the XOR gate using a neural network (NN). The various NN parameter sets are randomly sampled.

Setup NN

seed = 100

import torch
import torch.nn as nn
import pandas as pd
import numpy as np

class XOR(nn.Module):
    def __init__(self):
        super(XOR, self).__init__()

        self.layers = nn.Sequential(
            nn.Linear(2, 2),
            nn.Tanh(),
            nn.Linear(2, 1),
        )
    def forward(self, x):
        return self.layers(x)

def set_weights_biases(model, weights):
    with torch.no_grad():
        model.layers[0].weight.copy_(torch.tensor([
            [weights[0], weights[1]],
            [weights[2], weights[3]]
        ]))
        model.layers[0].bias.copy_(torch.tensor([weights[4], weights[5]]))
        model.layers[2].weight.copy_(torch.tensor([[weights[6], weights[7]]]))
        model.layers[2].bias.copy_(torch.tensor([weights[8]]))

device = torch.device("cpu")

x_train = torch.tensor([[0,0], [0,1], [1,1], [1,0]], device=device).float()
y_train = torch.tensor([[0], [1], [0], [1]], device=device).float()


x_val = torch.clone(x_train)
y_val = torch.clone(y_train)

loss_fn = nn.BCEWithLogitsLoss()

Setup pyXla sample

from pyxla.sampling import RandomSampler
from pyxla import load_data
import pyxla
import math

def loss_as_objective(weights):
    model = XOR()
    model.eval()
    set_weights_biases(model, weights)
    pred = model(x_train)
    loss = loss_fn(pred, y_train)
    return loss.item()

std1 = math.sqrt(2.0 / float(2 + 2))
std2 = math.sqrt(2.0 / float(2 + 1))

# sample as close as possible to the starting point of SGD setup in PyTorch
rng = np.random.default_rng(seed=seed)

X_wts1 = pd.DataFrame(rng.normal(0, std1, (1000, 4)))
X_wts2 = pd.DataFrame(rng.normal(0, std2, (1000, 2)))
X_bs = RandomSampler(sample_size=1000, dim=3, l_bound=0, u_bound=1, seed=seed, return_neighbourhood=False, representation='continuous').sample()
X_wts1.columns = ['w0', 'w1', 'w2', 'w3']
X_wts2.columns = ['w4', 'w5']
X_bs.columns = ['b0', 'b1', 'b2']

X = pd.concat([X_wts1, X_wts2, X_bs], axis=1)

sample = {
    'name': 'xor_nn',
    'X': X,
    'F': loss_as_objective,
    'D': 'euclidean', # use any metric supported by scipy
    'N': 'hilbert-curve'
}
load_data(sample)

feat, plot = pyxla.distr_f(sample, title=False)

plot.savefig('distr-f-xor-nn-sampled.png', dpi=300)

corr, imp, plot = pyxla.X_imp(sample, estimator='ridge', n_repeats=30, seed=42, suptitle=False)

plot.savefig('x-imp-xor-nn-sampled.png', dpi=300)

corr, plot = pyxla.fdc(sample)

plot.savefig('fdc-xor-nn-sampled.png', dpi=300)

corr, plot = pyxla.rdc(sample)

corr, plot = pyxla.pdc(sample)

corr, plot = pyxla.disp_best(sample)

/home/toni/Projects/pyxla-wg/src/pyxla/__init__.py:1409: RuntimeWarning: divide by zero encountered in log
  forward = lambda x: np.log(x / init_percentage) / np.log(growth_factor)

corr, plot = pyxla.nfc(sample)

corr, plot = pyxla.nrc(sample)