2-One_layer_encode_with_square_sin.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook shows that how square wave and sin wave embedder works with different number of samples when embedding dimension is large enough."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pylab import *\n",
    "from skimage.color import rgb2gray\n",
    "from skimage.transform import rescale,resize\n",
    "from torch.autograd import Variable\n",
    "import torch, time,os\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import scipy\n",
    "\n",
    "\n",
    "# Set the CUDA flag\n",
    "cuda_flag = True\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "criterion = torch.nn.MSELoss()\n",
    "epochs = 4000\n",
    "lr = 1e-3\n",
    "\n",
    "\n",
    "sample_N = 512\n",
    "sample_ds = [1,2,4,8,16]\n",
    "fs = [1,5,10,20,50,100,200]\n",
    "embedding_dimension = 10000\n",
    "\n",
    "Nf = len(fs)\n",
    "Nd = len(sample_ds)\n",
    "N_test = 1\n",
    "N_repeat = 1\n",
    "\n",
    "#kk = torch.randint(512,(N_test,))\n",
    "kk = [158, 329,  47, 344, 257, 410, 426, 199, 426, 295]\n",
    "#kk = [ 88, 269, 409,  59, 245, 382, 277, 362,  23, 308, 504, 448, 411, 304, 365, 200,  92, 350, 405,  77]\n",
    "\n",
    "losses_ = np.zeros((Nd,Nf,N_test,N_repeat,epochs))\n",
    "sr_ = np.zeros((Nd,Nf))\n",
    "trn_psnr_ = np.zeros((Nd,Nf,N_test,N_repeat,epochs))\n",
    "tst_psnr_ = np.zeros((Nd,Nf,N_test,N_repeat,epochs))\n",
    "\n",
    "file_path = 'Final_2one/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rgb = imread('4.2.07.tiff');im = rgb2gray(rgb);\n",
    "signal = torch.from_numpy(im).type(torch.FloatTensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "width = 256\n",
    "\n",
    "# Here define the MLP\n",
    "class SLP(nn.Module):\n",
    "    def __init__(self,input_dim):\n",
    "        super(SLP, self).__init__()\n",
    "        self.fco = nn.Linear(input_dim, 1)\n",
    "    def forward(self, x):\n",
    "        out = self.fco(x)\n",
    "        return out\n",
    "    def name(self):\n",
    "        return \"SLP\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def srank_func(X):\n",
    "    # Get the SVD\n",
    "    (u,s,v) = torch.svd(X)\n",
    "    sr = (s*s).sum()/s[0]/s[0]\n",
    "    return sr\n",
    "\n",
    "def psnr_func(x,y):\n",
    "    diff = x - y\n",
    "    err = (diff*diff).flatten().mean().log10()\n",
    "    return -10*err\n",
    "\n",
    "def impulse_embedding(x, b, s=1e-4):\n",
    "    emb = (abs(x-b)<s).type(torch.FloatTensor)\n",
    "    return emb/emb.norm(dim=1).max()\n",
    "\n",
    "def square_embedding(x, b, f=1):\n",
    "    emb = (torch.sin(2.*np.pi*f*(x-b))>0).type(torch.FloatTensor)\n",
    "    return emb/emb.norm(dim=1).max()\n",
    "\n",
    "def shift_embedding(x, c):\n",
    "    all_emb = scipy.linalg.circulant(c)\n",
    "    idx = (x*all_emb.shape[0]).int().squeeze()\n",
    "    emb = torch.from_numpy(all_emb[idx]).type(torch.FloatTensor)\n",
    "    return emb/emb.norm(dim=1).max()\n",
    "    \n",
    "\n",
    "def norm_func(x):\n",
    "    # Flatten the data \n",
    "    #x = x.flatten(1,3)\n",
    "    \n",
    "    # Normalize for gain and bias\n",
    "    y = x-x.mean(1).unsqueeze(-1)\n",
    "    y = x/x.norm(dim=1).unsqueeze(-1)    \n",
    "    return y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# square wave embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = torch.from_numpy(np.linspace(0, 1, sample_N+1)[:-1].reshape(-1,1)).type(torch.FloatTensor)\n",
    "\n",
    "\n",
    "for i in range(Nd):\n",
    "    sample_d = sample_ds[i]\n",
    "    \n",
    "    for ff in range(Nf):\n",
    "        f0 = fs[ff]\n",
    "        \n",
    "        c = np.sin(2.*np.pi*f0*np.linspace(0, 1, embedding_dimension+1)[:-1])>0\n",
    "        c = c/np.linalg.norm(c)\n",
    "        \n",
    "        train_data = shift_embedding(all_data[::2*sample_d],c)\n",
    "        test_data = shift_embedding(all_data[sample_d::2*sample_d],c)\n",
    "\n",
    "        # Calculate the input stable rank\n",
    "        sr_[i,ff] = srank_func(train_data)\n",
    "\n",
    "\n",
    "        for j in range(N_test):\n",
    "            start_time = time.time()\n",
    "\n",
    "\n",
    "            # Prepare the data\n",
    "            all_target = signal[kk[j]].reshape(-1,1)\n",
    "            train_label = all_target[::2*sample_d]\n",
    "            test_label = all_target[sample_d::2*sample_d]\n",
    "\n",
    "            xt, yt = train_data.cuda(),train_label.cuda()\n",
    "            xt, yt = Variable(xt), Variable(yt)\n",
    "\n",
    "            xts, yts = test_data.cuda(),test_label.cuda()\n",
    "\n",
    "\n",
    "            for k in range(N_repeat):\n",
    "                # Initialize classification model to learn\n",
    "                model = SLP(embedding_dimension).cuda()\n",
    "                # Set the optimization\n",
    "                optimizer = optim.Adam(model.parameters(), lr, betas=(0.9, 0.999),weight_decay=1e-8)\n",
    "\n",
    "\n",
    "                for epoch in range(epochs):\n",
    "                    model.train()\n",
    "                    optimizer.zero_grad()\n",
    "\n",
    "                    out = model(xt)\n",
    "                    loss = criterion(out, yt)\n",
    "\n",
    "                    loss.backward()\n",
    "                    losses_[i,ff,j,k,epoch] = loss.item()\n",
    "                    optimizer.step()\n",
    "\n",
    "                    model.eval()\n",
    "                    trn_psnr_[i,ff,j,k,epoch] = psnr_func(model(xt),yt)\n",
    "                    tst_psnr_[i,ff,j,k,epoch] = psnr_func(model(xts),yts)\n",
    "\n",
    "\n",
    "                print(\"==>>> N: %g, F: %g, train psnr: %g--- , test psnr: %g--- , time: %g seconds ---\" \n",
    "                        % (i,ff, np.mean(trn_psnr_[i,ff,j,k,-100:]),np.mean(tst_psnr_[i,ff,j,k,-100:]),time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(Nf):\n",
    "    figure()\n",
    "    for j in range(Nd):\n",
    "        plot(np.mean(trn_psnr_[j,i,:,:],(0,1)))\n",
    "        plot(np.mean(tst_psnr_[j,i,:,:],(0,1)),'--')\n",
    "\n",
    "figure()\n",
    "for i in range(Nf):\n",
    "    plot(sample_ds,sr_[:,i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = 'square'\n",
    "np.save(file_path+file_name+'_losses',losses_)\n",
    "np.save(file_path+file_name+'_sr',sr_)\n",
    "np.save(file_path+file_name+'_trn',trn_psnr_)\n",
    "np.save(file_path+file_name+'_tst',tst_psnr_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sin embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = torch.from_numpy(np.linspace(0, 1, sample_N+1)[:-1].reshape(-1,1)).type(torch.FloatTensor)\n",
    "\n",
    "\n",
    "for i in range(Nd):\n",
    "    sample_d = sample_ds[i]\n",
    "    \n",
    "    for ff in range(Nf):\n",
    "        f0 = fs[ff]\n",
    "        \n",
    "        c = np.sin(2.*np.pi*f0*np.linspace(0, 1, embedding_dimension+1)[:-1])\n",
    "        c = c/np.linalg.norm(c)\n",
    "        \n",
    "        train_data = shift_embedding(all_data[::2*sample_d],c)\n",
    "        test_data = shift_embedding(all_data[sample_d::2*sample_d],c)\n",
    "\n",
    "        # Calculate the input stable rank\n",
    "        sr_[i,ff] = srank_func(train_data)\n",
    "\n",
    "\n",
    "        for j in range(N_test):\n",
    "            start_time = time.time()\n",
    "\n",
    "\n",
    "            # Prepare the data\n",
    "            all_target = signal[kk[j]].reshape(-1,1)\n",
    "            train_label = all_target[::2*sample_d]\n",
    "            test_label = all_target[sample_d::2*sample_d]\n",
    "\n",
    "            xt, yt = train_data.cuda(),train_label.cuda()\n",
    "            xt, yt = Variable(xt), Variable(yt)\n",
    "\n",
    "            xts, yts = test_data.cuda(),test_label.cuda()\n",
    "\n",
    "\n",
    "            for k in range(N_repeat):\n",
    "                # Initialize classification model to learn\n",
    "                model = SLP(embedding_dimension).cuda()\n",
    "                # Set the optimization\n",
    "                optimizer = optim.Adam(model.parameters(), lr, betas=(0.9, 0.999),weight_decay=1e-8)\n",
    "\n",
    "\n",
    "                for epoch in range(epochs):\n",
    "                    model.train()\n",
    "                    optimizer.zero_grad()\n",
    "\n",
    "                    out = model(xt)\n",
    "                    loss = criterion(out, yt)\n",
    "\n",
    "                    loss.backward()\n",
    "                    losses_[i,ff,j,k,epoch] = loss.item()\n",
    "                    optimizer.step()\n",
    "\n",
    "                    model.eval()\n",
    "                    trn_psnr_[i,ff,j,k,epoch] = psnr_func(model(xt),yt)\n",
    "                    tst_psnr_[i,ff,j,k,epoch] = psnr_func(model(xts),yts)\n",
    "\n",
    "\n",
    "                print(\"==>>> N: %g, F: %g, train psnr: %g--- , test psnr: %g--- , time: %g seconds ---\" \n",
    "                        % (i,ff, np.mean(trn_psnr_[i,ff,j,k,-100:]),np.mean(tst_psnr_[i,ff,j,k,-100:]),time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(Nf):\n",
    "    figure()\n",
    "    for j in range(Nd):\n",
    "        plot(np.mean(trn_psnr_[j,i,:,:],(0,1)))\n",
    "        plot(np.mean(tst_psnr_[j,i,:,:],(0,1)),'--')\n",
    "\n",
    "figure()\n",
    "for i in range(Nf):\n",
    "    plot(sample_ds,sr_[:,i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = 'sin'\n",
    "np.save(file_path+file_name+'_losses',losses_)\n",
    "np.save(file_path+file_name+'_sr',sr_)\n",
    "np.save(file_path+file_name+'_trn',trn_psnr_)\n",
    "np.save(file_path+file_name+'_tst',tst_psnr_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "signal = torch.randn((512,512)).type(torch.FloatTensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pylab import *\n",
    "from skimage.color import rgb2gray\n",
    "from skimage.transform import rescale,resize\n",
    "from torch.autograd import Variable\n",
    "import torch, time,os\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import scipy\n",
    "\n",
    "\n",
    "# Set the CUDA flag\n",
    "cuda_flag = True\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "criterion = torch.nn.MSELoss()\n",
    "epochs = 4000\n",
    "lr = 1e-3\n",
    "\n",
    "\n",
    "sample_N = 512\n",
    "sample_ds = [1,2,3,4,6,8,11,16]\n",
    "fs = [1]\n",
    "embedding_dimension = 10000\n",
    "\n",
    "Nf = len(fs)\n",
    "Nd = len(sample_ds)\n",
    "N_test = 1\n",
    "N_repeat = 1\n",
    "\n",
    "#kk = torch.randint(512,(N_test,))\n",
    "kk = [158, 329,  47, 344, 257, 410, 426, 199, 426, 295]\n",
    "#kk = [ 88, 269, 409,  59, 245, 382, 277, 362,  23, 308, 504, 448, 411, 304, 365, 200,  92, 350, 405,  77]\n",
    "\n",
    "losses_ = np.zeros((Nd,Nf,N_test,N_repeat,epochs))\n",
    "sr_ = np.zeros((Nd,Nf))\n",
    "trn_psnr_ = np.zeros((Nd,Nf,N_test,N_repeat,epochs))\n",
    "tst_psnr_ = np.zeros((Nd,Nf,N_test,N_repeat,epochs))\n",
    "\n",
    "file_path = 'Final_2one/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = torch.from_numpy(np.linspace(0, 1, sample_N+1)[:-1].reshape(-1,1)).type(torch.FloatTensor)\n",
    "\n",
    "\n",
    "for i in range(Nd):\n",
    "    sample_d = sample_ds[i]\n",
    "    \n",
    "    for ff in range(Nf):\n",
    "        f0 = fs[ff]\n",
    "        \n",
    "        c = np.sin(2.*np.pi*f0*np.linspace(0, 1, embedding_dimension+1)[:-1])>0\n",
    "        c = c/np.linalg.norm(c)\n",
    "        \n",
    "        train_data = shift_embedding(all_data[::2*sample_d],c)\n",
    "        test_data = shift_embedding(all_data[sample_d::2*sample_d],c)\n",
    "\n",
    "        # Calculate the input stable rank\n",
    "        sr_[i,ff] = srank_func(train_data)\n",
    "\n",
    "\n",
    "        for j in range(N_test):\n",
    "            start_time = time.time()\n",
    "\n",
    "\n",
    "            # Prepare the data\n",
    "            all_target = signal[kk[j]].reshape(-1,1)\n",
    "            train_label = all_target[::2*sample_d]\n",
    "            test_label = all_target[sample_d::2*sample_d]\n",
    "\n",
    "            xt, yt = train_data.cuda(),train_label.cuda()\n",
    "            xt, yt = Variable(xt), Variable(yt)\n",
    "\n",
    "            xts, yts = test_data.cuda(),test_label.cuda()\n",
    "\n",
    "\n",
    "            for k in range(N_repeat):\n",
    "                # Initialize classification model to learn\n",
    "                model = SLP(embedding_dimension).cuda()\n",
    "                # Set the optimization\n",
    "                optimizer = optim.Adam(model.parameters(), lr, betas=(0.9, 0.999),weight_decay=1e-8)\n",
    "\n",
    "\n",
    "                for epoch in range(epochs):\n",
    "                    model.train()\n",
    "                    optimizer.zero_grad()\n",
    "\n",
    "                    out = model(xt)\n",
    "                    loss = criterion(out, yt)\n",
    "\n",
    "                    loss.backward()\n",
    "                    losses_[i,ff,j,k,epoch] = loss.item()\n",
    "                    optimizer.step()\n",
    "\n",
    "                    model.eval()\n",
    "                    trn_psnr_[i,ff,j,k,epoch] = psnr_func(model(xt),yt)\n",
    "                    tst_psnr_[i,ff,j,k,epoch] = psnr_func(model(xts),yts)\n",
    "\n",
    "\n",
    "                print(\"==>>> N: %g, F: %g, train psnr: %g--- , test psnr: %g--- , time: %g seconds ---\" \n",
    "                        % (i,ff, np.mean(trn_psnr_[i,ff,j,k,-100:]),np.mean(tst_psnr_[i,ff,j,k,-100:]),time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(Nf):\n",
    "    figure()\n",
    "    for j in range(Nd):\n",
    "        plot(np.mean(trn_psnr_[j,i,:,:],(0,1)))\n",
    "        plot(np.mean(tst_psnr_[j,i,:,:],(0,1)),'--')\n",
    "\n",
    "figure()\n",
    "for i in range(Nf):\n",
    "    plot(sample_ds,sr_[:,i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = 'square_random'\n",
    "np.save(file_path+file_name+'_losses',losses_)\n",
    "np.save(file_path+file_name+'_sr',sr_)\n",
    "np.save(file_path+file_name+'_trn',trn_psnr_)\n",
    "np.save(file_path+file_name+'_tst',tst_psnr_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = torch.from_numpy(np.linspace(0, 1, sample_N+1)[:-1].reshape(-1,1)).type(torch.FloatTensor)\n",
    "\n",
    "\n",
    "for i in range(Nd):\n",
    "    sample_d = sample_ds[i]\n",
    "    \n",
    "    for ff in range(Nf):\n",
    "        f0 = fs[ff]\n",
    "        \n",
    "        c = np.sin(2.*np.pi*f0*np.linspace(0, 1, embedding_dimension+1)[:-1])\n",
    "        c = c/np.linalg.norm(c)\n",
    "        \n",
    "        train_data = shift_embedding(all_data[::2*sample_d],c)\n",
    "        test_data = shift_embedding(all_data[sample_d::2*sample_d],c)\n",
    "\n",
    "        # Calculate the input stable rank\n",
    "        sr_[i,ff] = srank_func(train_data)\n",
    "\n",
    "\n",
    "        for j in range(N_test):\n",
    "            start_time = time.time()\n",
    "\n",
    "\n",
    "            # Prepare the data\n",
    "            all_target = signal[kk[j]].reshape(-1,1)\n",
    "            train_label = all_target[::2*sample_d]\n",
    "            test_label = all_target[sample_d::2*sample_d]\n",
    "\n",
    "            xt, yt = train_data.cuda(),train_label.cuda()\n",
    "            xt, yt = Variable(xt), Variable(yt)\n",
    "\n",
    "            xts, yts = test_data.cuda(),test_label.cuda()\n",
    "\n",
    "\n",
    "            for k in range(N_repeat):\n",
    "                # Initialize classification model to learn\n",
    "                model = SLP(embedding_dimension).cuda()\n",
    "                # Set the optimization\n",
    "                optimizer = optim.Adam(model.parameters(), lr, betas=(0.9, 0.999),weight_decay=1e-8)\n",
    "\n",
    "\n",
    "                for epoch in range(epochs):\n",
    "                    model.train()\n",
    "                    optimizer.zero_grad()\n",
    "\n",
    "                    out = model(xt)\n",
    "                    loss = criterion(out, yt)\n",
    "\n",
    "                    loss.backward()\n",
    "                    losses_[i,ff,j,k,epoch] = loss.item()\n",
    "                    optimizer.step()\n",
    "\n",
    "                    model.eval()\n",
    "                    trn_psnr_[i,ff,j,k,epoch] = psnr_func(model(xt),yt)\n",
    "                    tst_psnr_[i,ff,j,k,epoch] = psnr_func(model(xts),yts)\n",
    "\n",
    "\n",
    "                print(\"==>>> N: %g, F: %g, train psnr: %g--- , test psnr: %g--- , time: %g seconds ---\" \n",
    "                        % (i,ff, np.mean(trn_psnr_[i,ff,j,k,-100:]),np.mean(tst_psnr_[i,ff,j,k,-100:]),time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(Nf):\n",
    "    figure()\n",
    "    for j in range(Nd):\n",
    "        plot(np.mean(trn_psnr_[j,i,:,:],(0,1)))\n",
    "        plot(np.mean(tst_psnr_[j,i,:,:],(0,1)),'--')\n",
    "\n",
    "figure()\n",
    "for i in range(Nf):\n",
    "    plot(sample_ds,sr_[:,i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = 'sin_random'\n",
    "np.save(file_path+file_name+'_losses',losses_)\n",
    "np.save(file_path+file_name+'_sr',sr_)\n",
    "np.save(file_path+file_name+'_trn',trn_psnr_)\n",
    "np.save(file_path+file_name+'_tst',tst_psnr_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}