Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Course README.md #17

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Add Course README.md #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8a584d9
Add Course README.md
R-Mahmoudi Feb 21, 2023
fc308b8
Created using Colaboratory
R-Mahmoudi Mar 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
246 changes: 246 additions & 0 deletions A_Implemention_Hintons_Forward_Forward_Algorithm_by_Pytorch.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,246 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"authorship_tag": "ABX9TyNY9rI/YuyhTw18U6jZV0fc"
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
},
"accelerator": "GPU",
"gpuClass": "standard"
},
"cells": [
{
"cell_type": "markdown",
"source": [
"##A Implemention Hintons Forward-Forward Algorithm\n",
"\n",
"\n",
"\n"
],
"metadata": {
"id": "H1GWQHX4CPfq"
}
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"id": "_butDyfHBSHi"
},
"outputs": [],
"source": [
"import torch\n",
"import torch.nn as nn\n",
"from tqdm import tqdm\n",
"from torch.optim import Adam\n",
"from torchvision.datasets import MNIST\n",
"from torch.utils.data import DataLoader\n",
"from torchvision.transforms import Compose,Normalize,ToTensor,Lambda"
]
},
{
"cell_type": "code",
"source": [
"def loaders(train_batch_size=50000, test_batch_size=10000):\n",
" transform = Compose([ToTensor(),Normalize((0.1307,), (0.3081,)),Lambda(lambda x: torch.flatten(x))])\n",
"\n",
" train_loader = DataLoader(MNIST('./data/', train=True,\n",
" download=True,\n",
" transform=transform),\n",
" batch_size=train_batch_size, shuffle=True)\n",
" \n",
" test_loader = DataLoader(\n",
" MNIST('./data/', train=False,\n",
" download=True,\n",
" transform=transform),\n",
" batch_size=test_batch_size, shuffle=False)\n",
"\n",
" return train_loader, test_loader"
],
"metadata": {
"id": "3ou_2dSDDfok"
},
"execution_count": 2,
"outputs": []
},
{
"cell_type": "code",
"source": [
"def overlay_y_on_x(x, y):\n",
" x_ = x.clone()\n",
" x_[:, :10] *= 0.0\n",
" x_[range(x.shape[0]), y] = x.max()\n",
" return x_"
],
"metadata": {
"id": "s-bhEMmoFkJy"
},
"execution_count": 3,
"outputs": []
},
{
"cell_type": "code",
"source": [
"\n",
"class Net(torch.nn.Module):\n",
"\n",
" def __init__(self, dims):\n",
" super().__init__()\n",
" self.layers = []\n",
" for d in range(len(dims) - 1):\n",
" self.layers += [Layer(dims[d], dims[d + 1]).cuda()]\n",
"\n",
" def predict(self, x):\n",
" goodness_per_label = []\n",
" for label in range(10):\n",
" h = overlay_y_on_x(x, label)\n",
" goodness = []\n",
" for layer in self.layers:\n",
" h = layer(h)\n",
" goodness += [h.pow(2).mean(1)]\n",
" goodness_per_label += [sum(goodness).unsqueeze(1)]\n",
" goodness_per_label = torch.cat(goodness_per_label, 1)\n",
" return goodness_per_label.argmax(1)\n",
"\n",
" def train(self, x_pos, x_neg):\n",
" h_pos, h_neg = x_pos, x_neg\n",
" for i, layer in enumerate(self.layers):\n",
" print('Training layer..', i, '...')\n",
" h_pos, h_neg = layer.train(h_pos, h_neg)\n",
"\n"
],
"metadata": {
"id": "BkyOAx26GACi"
},
"execution_count": 4,
"outputs": []
},
{
"cell_type": "code",
"source": [
"class Layer(nn.Linear):\n",
" def __init__(self, in_features, out_features,\n",
" bias=True, device=None, dtype=None):\n",
" super().__init__(in_features, out_features, bias, device, dtype)\n",
" self.relu = torch.nn.ReLU()\n",
" self.opt = Adam(self.parameters(), lr=0.03)\n",
" self.threshold = 2.0\n",
" self.num_epochs = 1000\n",
"\n",
" def forward(self, x):\n",
" x_direction = x / (x.norm(2, 1, keepdim=True) + 1e-4)\n",
" return self.relu(\n",
" torch.mm(x_direction, self.weight.T) +\n",
" self.bias.unsqueeze(0))\n",
"\n",
" def train(self, x_pos, x_neg):\n",
" for i in tqdm(range(self.num_epochs)):\n",
" g_pos = self.forward(x_pos).pow(2).mean(1)\n",
" g_neg = self.forward(x_neg).pow(2).mean(1)\n",
" \n",
" loss = torch.log(1 + torch.exp(torch.cat([\n",
" -g_pos + self.threshold,\n",
" g_neg - self.threshold]))).mean()\n",
" self.opt.zero_grad()\n",
" \n",
" loss.backward()\n",
" self.opt.step()\n",
" return self.forward(x_pos).detach(), self.forward(x_neg).detach()\n"
],
"metadata": {
"id": "D2bE_NpXGq4Z"
},
"execution_count": 5,
"outputs": []
},
{
"cell_type": "code",
"source": [
"if __name__ == \"__main__\":\n",
" torch.manual_seed(1234)\n",
" train_loader, test_loader = loaders()\n",
"\n",
" net = Net([784, 500, 500])\n",
" x, y = next(iter(train_loader))\n",
" x, y = x.cuda(), y.cuda()\n",
" x_pos = overlay_y_on_x(x, y)\n",
" rnd = torch.randperm(x.size(0))\n",
" x_neg = overlay_y_on_x(x, y[rnd])\n",
" \n",
" \n",
" \n",
" net.train(x_pos, x_neg)\n",
"\n",
" print('train error:', 1.0 - net.predict(x).eq(y).float().mean().item())\n",
"\n",
" x_te, y_te = next(iter(test_loader))\n",
" x_te, y_te = x_te.cuda(), y_te.cuda()\n",
"\n",
" print('test error:', 1.0 - net.predict(x_te).eq(y_te).float().mean().item())"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "KDnN8rRMHPq4",
"outputId": "509d95f0-0cbe-499e-a001-eda39cf8a399"
},
"execution_count": 8,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Training layer.. 0 ...\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"100%|██████████| 1000/1000 [00:57<00:00, 17.47it/s]\n"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"Training layer.. 1 ...\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"100%|██████████| 1000/1000 [00:38<00:00, 26.07it/s]\n"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"train error: 0.06754004955291748\n",
"test error: 0.06840002536773682\n"
]
}
]
},
{
"cell_type": "code",
"source": [],
"metadata": {
"id": "S_3Zya99IjM0"
},
"execution_count": null,
"outputs": []
}
]
}
3 changes: 3 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,9 @@ In this repository, I will share some useful notes and references about deployin
- [MLOPs-Zoomcamp](https://github.com/DataTalksClub/mlops-zoomcamp) [Great]
- [A collection of resources to learn about MLOPs](https://github.com/dair-ai/MLOPs-Primer) [Great]
- [MLEM: package and deploy machine learning models](https://github.com/iterative/mlem)
- [Full Stack Deep Learning](https://www.youtube.com/watch?v=-Iob-FW5jVM&list=PL1T8fO7ArWleMMI8KPJ_5D5XSlovTW_Ur)
- [Made With ML](https://madewithml.com/#mlops)
- [Machine Learning Engineering for Production (MLOps)](https://www.youtube.com/playlist?list=PLkDaE6sCZn6GMoA0wbpJLi3t34Gd8l0aK)

## Other:
- [A Guide to Production Level Deep Learning](https://github.com/alirezadir/Production-Level-Deep-Learning)
Expand Down