import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.utils.data import DataLoader, Subset
import matplotlib.pyplot as plt

# Import wandb and initialize
import wandb

# Initialize wandb with a project name and configuration
wandb.init(
    project="hands_on_ml_demo",
    config={
        "learning_rate": 0.01,
        "epochs": 10,
        "batch_size": 32,
        "hidden_activation": "relu",
        "hidden_size": 128,
        "dataset": "CIFAR-10 (cats (0) vs dogs (1))",
        "model": "SimpleBinaryNet"
    }
)
config = wandb.config

wandb: Using wandb-core as the SDK backend.  Please refer to https://wandb.me/wandb-core for more information.
wandb: Currently logged in as: nilesh2797 to https://api.wandb.ai. Use `wandb login --relogin` to force relogin

print(*[f"{k}: {v}" for k, v in config.items()], sep="\n")

learning_rate: 0.01
epochs: 10
batch_size: 32
hidden_activation: relu
hidden_size: 128
dataset: CIFAR-10 (cats (0) vs dogs (1))
model: SimpleBinaryNet

# Setup Data Transformation
transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])

# Setup Target Transformation (0 for cats, 1 for dogs)
target_transform = lambda x: 0 if x == 3 else 1

# Load and Filter CIFAR-10 Data for Binary Classification (Ships vs Planes)
train_dataset_full = datasets.CIFAR10(root='./data', train=True, transform=transform, target_transform=target_transform, download=True)
test_dataset_full = datasets.CIFAR10(root='./data', train=False, transform=transform, target_transform=target_transform, download=True)

train_indices = [i for i, target in enumerate(train_dataset_full.targets) if target in [3, 5]]
test_indices = [i for i, target in enumerate(test_dataset_full.targets) if target in [3, 5]]

train_dataset = Subset(train_dataset_full, train_indices)
test_dataset = Subset(test_dataset_full, test_indices)

# Create Validation Dataset
train_size = int(0.8 * len(train_dataset))
val_size = len(train_dataset) - train_size
train_dataset, val_dataset = torch.utils.data.random_split(train_dataset, [train_size, val_size])

from torchvision.utils import make_grid

# Function to visualize sample images from the dataset
def visualize_samples(dataset, num_samples=16):
    # Get a subset of the dataset
    indices = torch.randperm(len(dataset))[:num_samples]
    samples = [dataset[i][0] for i in indices]
    
    # Create a grid of images
    grid = make_grid(samples, nrow=4, normalize=True)
    
    # Plot the grid
    plt.figure(figsize=(8, 8))
    plt.imshow(grid.permute(1, 2, 0))
    plt.title('Sample Images from the Dataset')
    plt.axis('off')
    plt.show()

# Visualize sample images from the training dataset
visualize_samples(train_dataset)

# Data Loaders
train_loader = DataLoader(train_dataset, batch_size=config.batch_size, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=config.batch_size, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False)

# Define Simple Neural Network
class SimpleBinaryNet(nn.Module):
    def __init__(self, input_size=3*32*32, hidden_size=128, hidden_activation='sigmoid'):
        super(SimpleBinaryNet, self).__init__()
        self.w1 = nn.Parameter(torch.randn(input_size, hidden_size) / input_size**0.5) # Shape: (3*32*32) x hidden_size
        self.b1 = nn.Parameter(torch.zeros(hidden_size)) # Shape: hidden_size
        self.w2 = nn.Parameter(torch.randn(hidden_size, 1) / hidden_size**0.5) # Shape: hidden_size
        self.b2 = nn.Parameter(torch.zeros(1)) # Shape: 1
        self.hidden_activation_fn = F.sigmoid if hidden_activation == 'sigmoid' else F.relu if hidden_activation == 'relu' else ValueError("Unknown activation function")

    def forward(self, x):
        '''Forward pass of the network
        Args:
            x: Input tensor of shape (batch_size, 3*32*32)
        Returns:
            logits: Raw logits of shape (batch_size, 1)
        '''
        x = x.view(x.size(0), -1) # Flatten the image, shape: (batch_size, 3*32*32)
        z1 = torch.matmul(x, self.w1) + self.b1 # Shape: (batch_size, hidden_size)
        a1 = self.hidden_activation_fn(z1) # Shape: (batch_size, hidden_size)
        z2 = torch.matmul(a1, self.w2) + self.b2 # Shape: (batch_size, 1)
        return z2
    
class NNLinearBinaryNet(nn.Module):
    def __init__(self, input_size=3*32*32, hidden_size=128, hidden_activation='sigmoid'):
        super(NNLinearBinaryNet, self).__init__()
        self.fc1 = nn.Linear(input_size, hidden_size)
        self.fc2 = nn.Linear(hidden_size, hidden_size // 2)
        self.fc3 = nn.Linear(hidden_size // 2, 1)
        self.hidden_activation_fn = F.sigmoid if hidden_activation == 'sigmoid' else F.relu if hidden_activation == 'relu' else ValueError("Unknown activation function")

    def forward(self, x):
        '''Forward pass of the network
        Args:
            x: Input tensor of shape (batch_size, 3*32*32)
        Returns:
            logits: Raw logits of shape (batch_size, 1)
        '''
        x = x.view(x.size(0), -1)  # Flatten the image, shape: (batch_size, 3*32*32)
        x = self.hidden_activation_fn(self.fc1(x)) # Shape: (batch_size, hidden_size)
        x = self.hidden_activation_fn(self.fc2(x))
        x = self.fc3(x)
        return x

if config.model == "SimpleBinaryNet":
    model = SimpleBinaryNet(input_size=3*32*32, hidden_size=config.hidden_size, hidden_activation=config.hidden_activation)
elif config.model == "NNLinearNet":
    model = NNLinearBinaryNet(input_size=3*32*32, hidden_size=config.hidden_size, hidden_activation=config.hidden_activation)
else:
    raise ValueError("Unknown model type")

# Use wandb.watch to log gradients and model parameters
wandb.watch(model, log="all", log_freq=100, log_graph=True)

wandb: logging graph, to disable use `wandb.watch(log_graph=False)`

# Define Loss Function and Optimizer
criterion = nn.BCEWithLogitsLoss()
optimizer = optim.SGD(model.parameters(), lr=config.learning_rate)

sample_batch = next(iter(train_loader))

from torchviz import make_dot
pred = model(sample_batch[0])
loss = criterion(pred, sample_batch[1].float().view(-1, 1))
make_dot(loss, params=dict(model.named_parameters()), show_attrs=False, show_saved=False)

from tqdm import tqdm

# Define Training and Evaluation Functions
def train(model, device, train_loader, optimizer, criterion, epoch):
    model.train()
    running_loss = 0.0
    running_num_batches = 0
    pbar = tqdm(enumerate(train_loader), total=len(train_loader), desc=f'Epoch {epoch}', dynamic_ncols=True)
    for batch_idx, (data, target) in pbar:
        data, target = data.to(device), target.to(device).float().unsqueeze(1)
        
        optimizer.zero_grad()
        output = model(data)
        loss = criterion(output, target)
        loss.backward()
        optimizer.step()

        running_loss += loss.item()
        running_num_batches += 1
        # Log loss per batch
        wandb.log({"train_loss": loss.item(), "epoch": epoch, "batch": batch_idx})
        pbar.set_postfix({'mean loss': running_loss / running_num_batches})
    
    avg_loss = running_loss / len(train_loader)
    print(f'Average training loss for epoch {epoch}: {avg_loss:.6f}')
    return avg_loss

def evaluate(model, device, data_loader, criterion, epoch, split="val"):
    model.eval()
    test_loss = 0
    correct = 0
    sample_images = []  # To collect sample images and predictions
    sample_preds = []
    sample_targets = []
    
    with torch.no_grad():
        for batch_idx, (data, target) in enumerate(data_loader):
            data, target = data.to(device), target.to(device).float().unsqueeze(1)
            output = model(data)
            test_loss += criterion(output, target).item()
            preds = (torch.sigmoid(output) >= 0.5).to(target.dtype)  # threshold at 0.5
            correct += preds.eq(target.byte()).sum().item()

            # Save first batch sample predictions for visualization
            if batch_idx == 0:
                sample_images.append(data.cpu())
                sample_preds.append(preds.cpu())
                sample_targets.append(target.cpu())
    
    test_loss /= len(data_loader)
    accuracy = 100. * correct / len(data_loader.dataset)
    print(f'\n{split.capitalize()} set: Average loss: {test_loss:.4f}, Accuracy: {accuracy:.2f}%\n')
    
    # Log evaluation metrics to wandb
    wandb.log({f"{split}_loss": test_loss, f"{split}_accuracy": accuracy, "epoch": epoch})
    
    # Log sample predictions (showing first batch of test set)
    if sample_images:
        # Log a list of sample images with predictions and targets as captions
        wandb.log({
            f"{split}_sample_predictions": [wandb.Image(img, caption=f"Pred: {'Dog' if pred.item() else 'Cat'}, Gold: {'Dog' if target.item() else 'Cat'}")
                                   for img, pred, target in zip(sample_images[0], sample_preds[0], sample_targets[0])]
        })
    
    return test_loss, accuracy

# Training and Evaluation Loop with wandb tracking
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
epochs = config.epochs

train_losses = []
test_accuracies = []

for epoch in range(1, epochs + 1):
    train_loss = train(model, device, train_loader, optimizer, criterion, epoch)
    val_loss, val_acc = evaluate(model, device, val_loader, criterion, epoch)
    train_loss, train_acc = evaluate(model, device, train_loader, criterion, epoch, split="train")
    
    wandb.log({"train_loss": train_loss, "train_accuracy": train_acc, "val_loss": val_loss, "val_accuracy": val_acc, "epoch": epoch})

Epoch 1: 100%|██████████| 250/250 [00:01<00:00, 243.80it/s, mean loss=0.664]

Average training loss for epoch 1: 0.663668

Val set: Average loss: 0.6542, Accuracy: 60.50%


Train set: Average loss: 0.6381, Accuracy: 63.21%

Epoch 2: 100%|██████████| 250/250 [00:00<00:00, 287.84it/s, mean loss=0.638]

Average training loss for epoch 2: 0.637992

Val set: Average loss: 0.6510, Accuracy: 61.95%


Train set: Average loss: 0.6206, Accuracy: 65.42%

Epoch 3: 100%|██████████| 250/250 [00:00<00:00, 296.58it/s, mean loss=0.624]

Average training loss for epoch 3: 0.624370

Val set: Average loss: 0.6471, Accuracy: 62.50%


Train set: Average loss: 0.6063, Accuracy: 66.96%

Epoch 4: 100%|██████████| 250/250 [00:00<00:00, 285.91it/s, mean loss=0.612]

Average training loss for epoch 4: 0.612083

Val set: Average loss: 0.6469, Accuracy: 62.25%


Train set: Average loss: 0.5976, Accuracy: 68.08%

Epoch 5: 100%|██████████| 250/250 [00:01<00:00, 230.08it/s, mean loss=0.603]

Average training loss for epoch 5: 0.603175

Val set: Average loss: 0.6406, Accuracy: 62.80%


Train set: Average loss: 0.5835, Accuracy: 69.89%

test_loss, test_acc = evaluate(model, device, test_loader, criterion, epoch, split="test")
wandb.finish()

Test set: Average loss: 0.6390, Accuracy: 64.10%

# Define the sweep configuration
sweep_configuration = {
    'method': 'grid',  # Bayesian optimization
    'metric': {
         'name': 'val_accuracy',
         'goal': 'maximize'
    },
    'parameters': {
         'learning_rate': {'values': [0.001, 0.0001, 0.01]},
         'batch_size': {'values': [32, 64, 128]},
         'hidden_size': {'values': [64, 128, 256]},
         'epochs': {'value': 3}  # keep epochs small for quick sweeps
    }
}

# Initialize the sweep
sweep_id = wandb.sweep(sweep=sweep_configuration, project="hands_on_ml_demo")
print("Sweep ID:", sweep_id)

Create sweep with ID: 1sn1nvxk
Sweep URL: https://wandb.ai/nilesh2797/hands_on_ml_demo/sweeps/1sn1nvxk
Sweep ID: 1sn1nvxk

def sweep_train():
    """Training function for hyperparameter sweep runs."""
    # Initialize wandb for this sweep run
    wandb.init()
    config = wandb.config
    
    # Set up data loaders using the hyperparameter batch size
    batch_size = config.batch_size
    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)
    
    # Initialize the model using the hyperparameter hidden_size
    model = SimpleBinaryNet(hidden_size=config.hidden_size)
    model.to(device)
    
    # Define loss function and optimizer with hyperparameter learning_rate
    criterion = nn.BCEWithLogitsLoss()
    optimizer = optim.SGD(model.parameters(), lr=config.learning_rate)
    
    # Run training for the specified number of epochs
    for epoch in range(1, config.epochs + 1):
         train_loss = train(model, device, train_loader, optimizer, criterion, epoch)
         val_loss, val_accuracy = evaluate(model, device, val_loader, criterion, epoch)
         
         # Log metrics for each epoch
         wandb.log({
             "epoch_train_loss": train_loss,
             "epoch_val_loss": val_loss,
             "val_accuracy": val_accuracy,
             "epoch": epoch
         })

    # Finish the run
    wandb.finish()

# Launch the sweep agent to run a specified number of runs
wandb.agent(sweep_id, function=sweep_train, count=None)

wandb: Agent Starting Run: 560ab4u8 with config:
wandb: 	batch_size: 32
wandb: 	epochs: 3
wandb: 	hidden_size: 64
wandb: 	learning_rate: 0.001

Epoch 1: 100%|██████████| 250/250 [00:00<00:00, 302.01it/s, mean loss=0.707]

Average training loss for epoch 1: 0.707201

Val set: Average loss: 0.6967, Accuracy: 49.25%

Epoch 2: 100%|██████████| 250/250 [00:00<00:00, 324.50it/s, mean loss=0.69]

Average training loss for epoch 2: 0.690431

Val set: Average loss: 0.6871, Accuracy: 54.60%

Epoch 3: 100%|██████████| 250/250 [00:01<00:00, 235.25it/s, mean loss=0.684]

Average training loss for epoch 3: 0.684309

Val set: Average loss: 0.6818, Accuracy: 57.10%

wandb: Agent Starting Run: ae5tv3br with config:
wandb: 	batch_size: 32
wandb: 	epochs: 3
wandb: 	hidden_size: 64
wandb: 	learning_rate: 0.0001

Epoch 1: 100%|██████████| 250/250 [00:00<00:00, 294.21it/s, mean loss=0.7]

Average training loss for epoch 1: 0.700002

Val set: Average loss: 0.7019, Accuracy: 49.30%

def get_gradient_norms(model):
    norms = {}
    for name, param in model.named_parameters():
        if param.grad is not None:
            norms[name] = param.grad.norm().item()
    return norms

# Example: Log gradient norms during training
grad_norms = get_gradient_norms(model)
wandb.log({"gradient_norms": grad_norms, "epoch": epoch})

---------------------------------------------------------------------------
Error                                     Traceback (most recent call last)
Cell In[350], line 10
      8 # Example: Log gradient norms during training
      9 grad_norms = get_gradient_norms(model)
---> 10 wandb.log({"gradient_norms": grad_norms, "epoch": epoch})

File ~/miniconda3/lib/python3.11/site-packages/wandb/sdk/lib/preinit.py:36, in PreInitCallable.<locals>.preinit_wrapper(*args, **kwargs)
     35 def preinit_wrapper(*args: Any, **kwargs: Any) -> Any:
---> 36     raise wandb.Error(f"You must call wandb.init() before {name}()")

Error: You must call wandb.init() before wandb.log()

def activation_hook(module, input, output):
    wandb.log({
        f"{module.__class__.__name__}_activation_min": output.min().item(),
        f"{module.__class__.__name__}_activation_max": output.max().item(),
        f"{module.__class__.__name__}_activation_mean": output.mean().item(),
    })

# Register the hook for the first fully connected layer
model.w1.register_forward_hook(activation_hook)

batch	▂▇▇▃▃▄▄▆▆▂▄▄▆█▂▅▆▆▁▂▄██▅▆▁▃▄▅▇▁▂▅▅▅▂▂▃▅▆
epoch	▁▁▂▂▂▂▃▃▃▃▃▃▃▄▄▄▄▄▅▅▅▆▆▆▆▆▆▆▆▆▆▆▇▇▇█████
test_accuracy	▁
test_loss	▁
train_accuracy	▁▁▂▂▃▃▅▅▅▅▅▅▇▇▇▇▇▇██
train_loss	▇▄▆▅▃▇▆▄▅▃▅▆▄▅▂▄▅▅▄▄▆▃█▃▃▆▃▄▁▃▂▄▁▃▄▄▆▂▅▃
val_accuracy	▁▁▅▅▂▂▆▆▇▇██▅▅▇▇▆▆▇▇
val_loss	██▄▄▅▅▃▃▂▂▁▁▃▃▁▁▄▄▃▃

batch	▂▃▃▄▅▆▆▇▇▇▇▇█▁▂▃▄▄▅▆▇▇███▁▁▃▃▃▅▅▅▅▆▆▆▇▇█
epoch	▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅███████████████
epoch_train_loss	█▃▁
epoch_val_loss	█▃▁
train_loss	▆██▆▅▁▃▁▇▂▂▅▆▄▂▄▃▄▃▃▅▃▂▄▃▄▁▄▃▄▃▂▃▃▃▅▂▄▂▄
val_accuracy	▁▁▆▆██
val_loss	█▃▁

batch	▁▂▃▃▃▄▅▆▆▇▇▇█▁▂▂▂▃▃▄▄▅▅▆▆▇██▁▁▂▃▄▄▄▅▆▇██
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅███████████████
epoch_train_loss	█▄▁
epoch_val_loss	█▄▁
train_loss	▆▇▃▅▆▇▂█▃▅▄▅▆▇█▄▁▄█▆▆▆▁▇▃▄▅▅▅▄▆▃▁▆▅▆▄█▆▃
val_accuracy	▁▁▃▃██
val_loss	█▄▁

batch	▁▁▂▂▃▄▄▅▆▆▇▇▇██▁▁▁▂▂▃▃▃▄▅▇▇▇█▂▂▃▄▄▅▆▆▇▇█
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅██████████████
epoch_train_loss	█▃▁
epoch_val_loss	█▄▁
train_loss	▆█▆▆▅█▆▅▆▆▇▇▃▆▅▅▅▄▄▅██▄▅▃█▁▂▇▄▇▅█▅▄▄▅▇▁▄
val_accuracy	▁▁▂▂██
val_loss	█▄▁

batch	▁▁▂▃▃▇▁▂▂▃▃▄▅▅▅▆▆▇▇▁▁▁▁▂▂▃▄▄▄▄▆▆▆▆▆▇▇▇▇█
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅█████████████
epoch_train_loss	█▃▁
epoch_val_loss	█▄▁
train_loss	▆▄█▅▃▇▇▄▅▅▄▄▅▂▆▃▄▅▃▆▅▄▅▄▅▄▃▃▃▄▄▃▂▃▂▂▁▄▆▂
val_accuracy	▁▁▅▅██
val_loss	█▄▁

batch	▂▂▃▃▄▄▄▅▅▇█████▂▂▃▄▄▄▄▄▄▄▅▇▇▇█▁▂▂▃▃▆▆▇▇█
epoch	▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅███████████
epoch_train_loss	█▄▁
epoch_val_loss	█▄▁
train_loss	▂▄▄▅▁█▅▅▃▆▅▆▆▆▆▅▆▇▆▄▅▄▆█▄▆▆▆▄▆▆▆▅▅▅▆▅▅▇▆
val_accuracy	▁▁██▇▇
val_loss	█▄▁

batch	▁▁▁▁▂▂▃▃▃▃▄▄▅▆▆▇▇▇█▁▂▂▂▃▃▃▄▅▅▆▇▇▇▇▇██▁▂█
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅████████████
epoch_train_loss	█▃▁
epoch_val_loss	█▂▁
train_loss	▅▆▇▆▇▄▃▅▅▅▄▅▄▇▁▆▅▃▃▄▅▁▇▄▃▄▅▄▆▄█▂▂▇▂▂▃▄█▄
val_accuracy	▁▁▃▃██
val_loss	█▂▁

batch	▁▂▂▃▄▅▅▅▆▇█▁▁▁▂▂▃▃▃▃▄▅▅▅▅▇▇█▁▁▄▅▅▆▇▇▇▇██
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅████████████
epoch_train_loss	█▃▁
epoch_val_loss	█▄▁
train_loss	█▄▄▅▃▃▄▃▄▃▃▄▂▄▂▂▃▂▃▄▃▂▃▃▁▂▂▃▃▄▂▃▃▂▃▃▅▄▄▁
val_accuracy	▁▁▆▆██
val_loss	█▄▁

batch	▁▂▂▂▃▄▄▅▆▇▇▇▁▁▁▂▄▅▅▆▆▇▇▇▇▁▁▁▂▂▂▂▃▃▃▅▆▇▇█
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅██████████████
epoch_train_loss	█▃▁
epoch_val_loss	█▃▁
train_loss	▆▆▂▅▅▆▃▅▄▇▇▅▆▆▁▇▂█▅▆▆▆▆▃▆▄▃▅▆▄▅▄▅▃▄▂▆▃▅▄
val_accuracy	▁▁▁▁██
val_loss	█▃▁

batch	▁▁▂▂▃▃▃▃▄▄▅▅▆▆▆▁▂▂▃▃▄▆▆▆▇▇█▂▂▃▃▄▅▅▆▆▇▇▇█
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅██████████
epoch_train_loss	█▃▁
epoch_val_loss	█▅▁
train_loss	▆▄▅▅▃▃▄▁▆▇▇▅▆▄▇▁▃▁▅▅▂▄▅█▂▃▅▃▄▅▂▄▄▅▃▃▁▂▅▁
val_accuracy	▁▁▁▁██
val_loss	█▅▁

batch	▁▂▂▃▃▃▃▃▄▆▁▂▂▂▃▃▄▄▅▅▇▇▇▇██▁▁▂▂▅▅▅▅▆▇▇███
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅████████████████
epoch_train_loss	█▃▁
epoch_val_loss	█▃▁
train_loss	▄█▆▄▂▁▃▇▃▆▂▆▄▃▄▁▄▄▃▂▃▁▃▁▃▂▂▃▂▃▃▁▂▂▁▂▂▃▂▂
val_accuracy	▁▁▂▂██
val_loss	█▃▁

batch	▁▂▂▂▃▅▅▅▆▆▇▇▇▇█▃▃▄▄▄▅▅▆▆██▂▃▃▃▄▅▅▅▅▅▆███
epoch	▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅████████████████
epoch_train_loss	█▄▁
epoch_val_loss	█▄▁
train_loss	█▄▇▅▆▅█▁▅▇▅▁▃▆▅▄▃▇▃█▃▇▆▅▅▁▅▇▅▅▆▄▆▄▄▆▅▄▄▁
val_accuracy	▁▁▁▁▁▁
val_loss	█▄▁

batch	▁▁▂▃▃▄▄▆▇▇██▁▂▂▃▃▃▃▃▄▄▅▆▇▇▇█▁▁▂▃▃▃▄▄▄▆▇█
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅█████████████
epoch_train_loss	█▂▁
epoch_val_loss	█▅▁
train_loss	▇█▄▄▃▃▃▃▃▃▂▃▃▂▄▂▃▁▂▂▂▃▃▄▃▁▁▅▂▂▂▄▁▄▄▁▂▂▁▁
val_accuracy	▁▁▇▇██
val_loss	█▅▁

batch	▁▂▃▃▄▅▅▅▆▆▆▆██▁▂▂▃▄▅▅▆▆▆▆▁▂▂▂▃▄▅▅▅▅▆▆▆▆█
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅███████████████
epoch_train_loss	█▂▁
epoch_val_loss	█▃▁
train_loss	▅█▃▄▂▂▆▂▄▃▂▁▂▃▃▂▂▁▃▂▁▂▃▂▁▂▂▂▂▂▁▂▂▁▂▂▁▂▁▂
val_accuracy	▁▁▄▄██
val_loss	█▃▁

batch	▁▂▂▂▃▃▃▃▄▄▄▅▆▆▆███▁▁▂▄▅▅▅▆▆▆▇▇▂▂▂▃▃▅▅▆▆█
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅▅██████████
epoch_train_loss	█▄▁
epoch_val_loss	█▄▁
train_loss	▅▅▇▆▆▆▅▄▄▃▅█▅▃▂▃▅▅▅▂▆▆▅▃▂▅▇▅▅▃▃▃▄▄▄▄▄▄▁▅
val_accuracy	▁▁▁▁▁▁
val_loss	█▄▁

batch	▁▂▃▃▄▆▇███▁▁▂▂▃▃▄▄▄▄▅▅▅▅▆▆▇▇███▁▂▃▃▄▅▆▆▇
epoch	▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅███████████████
epoch_train_loss	█▂▁
epoch_val_loss	█▃▁
train_loss	█▃▆▃▄▄▃▄▅▆▃▄▄▃▅▅▄▅▅▄▆▃▄▄▃▂▃▄▃▃▃▄▃▃▁▅▂▄▄▃
val_accuracy	▁▁▆▆██
val_loss	█▃▁

batch	▁▁▁▂▃▄▄▄▅▅▇▇▇▇▇▇██▂▂▃▃▃▄▄▅▆▆▆▆▇▇▇█▂▅▆▆▆█
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅████████████
epoch_train_loss	█▂▁
epoch_val_loss	█▄▁
train_loss	▅██▁▃▄▄▄▁▂▄▃▂▂▂▃▂▃▂▃▂▁▂▂▁▂▁▃▂▂▂▃▃▁▃▁▂▂▂▃
val_accuracy	▁▁▅▅██
val_loss	█▄▁

batch	▂▂▂▃▃▃▃▄▄▅▆▇███▂▃▃▃▄▄▄▅▆▆▇▇█▁▁▂▂▃▃▃▅▆▆▆▇
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅██████████
epoch_train_loss	█▄▁
epoch_val_loss	█▄▁
train_loss	▂▇▆▅█▆▅▇▆█▅▁▅▅▄▅▇▄▇▆▄▃▇▅▆▂▅▅▄▆▂▆▇▅▆▅▅▆▃▂
val_accuracy	▁▁▆▆██
val_loss	█▄▁

batch	▁▁▁▂▂▃▃▄▄▄▆▇███▂▂▂▂▃▃▄▄▄▅▆▇█▁▂▂▂▃▄▄▅▅▆▆█
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅██████████████
epoch_train_loss	█▃▁
epoch_val_loss	█▆▁
train_loss	▄▆▇▆▆▆▆▄▇▄▇▅▆▅▅▅▆▄▆▇▅▅▂▄▅▆▅▄▅▃▆▅▄█▄▃▁█▃▁
val_accuracy	▂▂▁▁██
val_loss	█▆▁

batch	▁▁▂▂▂▃▄▄▆▇▇▁▁▂▂▂▂▃▃▃▃▄▄▄▄▅▆▆▆▇▂▃▃▃▄▅▅▇▇█
epoch	▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅███████████
epoch_train_loss	█▄▁
epoch_val_loss	█▄▁
train_loss	▅█▇▇▄▆▃█▄▇▅▁▄▃▅▄▃▅▆▂▃▄▅▃▄▂▄▃▂▂▂▃▁▃▃▂▂▂▂▃
val_accuracy	▁▁▁▁██
val_loss	█▄▁

batch	▁▁▁▂▂▃▄▄▆▇█▁▁▁▂▂▂▃▃▄▄▅▆▆▇▇▂▂▂▂▃▃▃▃▄▆▆▆▇▇
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅████████
epoch_train_loss	█▃▁
epoch_val_loss	█▄▁
train_loss	▅▃▃▄▇▆▅▅▅▆▆▆▅▄▇▆▄▅▅▄▅█▅▃▅▂▃▃▅▅▂▅▃▆▅▁▄▄▃▇
val_accuracy	▁▁▅▅██
val_loss	█▄▁

Hands on Machine Learning¶

Life of an ML Experiment¶

Defining Problem¶

Defining Solution¶

Implementing Solution¶

Defining Problem¶

Demo problem: classify images of dogs vs cats¶

Key definitions of a problem¶

What is my input ($x$)?¶

What is my output ($y$)?¶

How do I make a collection of such inputs and outputs a.k.a create my dataset ($\mathcal{D}$)?¶

How do I measure success?¶

Exercise problem: build chatgpt?¶

What is my input ($x$)?¶

What is my output ($y$)?¶

How do I make a collection of such inputs and outputs a.k.a create my dataset ($\mathcal{D}$)?¶

How do I measure success?¶

Defining Solution¶

How do I represent my input ($x$) in numbers?¶

How do I represent my output ($y$) in numbers?¶

How do I map my input to my output a.k.a what is my model ($\mathcal{M}$)?¶

How do I measure success in a differentiable way a.k.a what is my loss function ($\mathcal{L}$)?¶

How do I train my model to minimize the loss a.k.a what is my optimizer ($\mathcal{O}$)?¶

Setting up the Environment¶

Torch Imports¶

WandB¶

Data preparation¶

Dataloader¶

Setting up Model, Loss, and Optimizer¶

Setting up Training and Evaluation Loops¶

Training the Model¶

Evaluating the Model¶

Run history:

Run summary:

Automatic Hyperparameter Sweep with wandb¶

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

batch	▁▂▂▃▃▃▃▃▃▅▅▅▅▆▆▆▇▇█▁▃▃▃▃▄▆▆▇▇▁▂▂▄▄▄▆▆▇▇█
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅██████████
epoch_train_loss	█▃▁
epoch_val_loss	█▄▁
train_loss	▅▅▆▆█▆▆▄▅▅▄▃▇▄▄▆▂▅▆▄▆▃▄▇█▄▄▄▄█▆▅▂▆▆▃▅▁▅▅
val_accuracy	▁▁▅▅██
val_loss	█▄▁

batch	▁▂▂▃▄▄▅▅▆▆▇▇▇██▂▃▃▃▄▅▅▆▆▇▁▁▂▃▃▅▅▅▆▆▇▇▇██
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅█████████████
epoch_train_loss	█▄▁
epoch_val_loss	█▄▁
train_loss	▃▆▆▆▄▄█▃▅▄▂▅▅▆▃▄▆▅█▃▄▅▂▆▃▄▃▃▃▆▇▅▃▁▄▃▄▇▂▃
val_accuracy	▁▁▇▇██
val_loss	█▄▁

batch	▁▂▂▃▃▄▄▄▅▅▅▆▇██▂▂▂▂▃▃▄▄▄▆▇▇██▁▂▂▂▂▃▄▄▄▅▇
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅█████████████
epoch_train_loss	█▄▁
epoch_val_loss	█▄▁
train_loss	▃▇▅█▅▂▁▃▂▃▅▃▅▅▆▆▆▅▅▅▃▁▆▅▄▃▄▄▆▃▂▅▇▃▆▃▅▆▅▆
val_accuracy	▁▁▁▁▁▁
val_loss	█▄▁

batch	▁▁▂▂▃▅▅▆▆▆▇██▁▂▃▃▄▄▅▇▇██▁▁▂▂▂▃▄▄▅▅▅▆▆▇▇█
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅████████████
epoch_train_loss	█▃▁
epoch_val_loss	█▃▁
train_loss	▆▆▅▅▅▅▆▇▅▃▄▄▄▄▅▄▄▆█▂▅▅▅▄▄▄▅▅▂▄▁▅▂▁▂▃▄▁▃▅
val_accuracy	▁▁████
val_loss	█▃▁

batch	▁▁▁▂▃▃▄▄▄▅▆▆▇▇▇▁▂▂▂▃▄▅▅▆▆▆▇▇██▂▃▅▅▅▆▆▆▇█
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅█████████████
epoch_train_loss	█▃▁
epoch_val_loss	█▄▁
train_loss	▃▅▄▆▅▄▄▅▅▅▅▁▃▃▄█▃▄▄▄▅▄▃▄▄▄▃▄▃▃▃▃▃▃▂▃▂▃▂▄
val_accuracy	▁▁▆▆██
val_loss	█▄▁

batch	▁▂▂▃▃▃▃▃▄▆▆▇▇▇▁▂▃▃▃▄▆▇██▁▂▂▃▃▃▅▅▆▆▆▇▇▇▇█
epoch	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅██████████
epoch_train_loss	█▄▁
epoch_val_loss	█▄▁
train_loss	▄█▅▄▃▅▄▅▃▂▃▆▃▅▇▄▄▄▄▁▄▂▂▅▃▄▃▅▂▃▂▃▅▅▅▃▃▅▇▂
val_accuracy	▁▁▅▅██
val_loss	█▄▁

batch	▁▁▂▂▃▃▄▄▆▇██▁▁▁▂▃▅▅▅▆▆▁▂▂▃▃▃▃▄▆▆▆▆▆▇▇▇██
epoch	▁▁▁▁▁▁▁▁▁▁▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅█████████████
epoch_train_loss	█▃▁
epoch_val_loss	█▃▁
train_loss	█▅▄▄▅▃▃▃▅▃▄▄▄▄▂▃▃▂▂▂▃▄▄▂▂▃▃▃▃▄▂▃▂▁▃▃▃▂▂▂
val_accuracy	▁▁██▇▇
val_loss	█▃▁