{
 "metadata": {
  "name": "",
  "signature": "sha256:d35f66e241c83fce49bf0f825461c9bef7321fa8529d41a1200b86c1371dbb65"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Chapter 11 - Information theory"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 1 - pg 488"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the information content in all symbols\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "Px_1=1./2;#probability 1\n",
      "Px_2=1./4;#probability 2\n",
      "Px_3=1./8;#probability 3\n",
      "Px_4=1./8;#probability 4\n",
      "\n",
      "#calculations\n",
      "Ix_1 = log2(1/(Px_1))#information content in first probability\n",
      "Ix_2 = log2(1/(Px_2))#information content in first probability\n",
      "Ix_3 = log2(1/(Px_3))#information content in first probability\n",
      "Ix_4 = log2(1/(Px_3))#information content in first probability\n",
      "\n",
      "#results\n",
      "print \"i. Information content of first symbol (bits) = \",Ix_1\n",
      "print \"ii. Information content of second symbol (bits) = \",Ix_2\n",
      "print \"iii. Information content of third  symbol (bits) = \",Ix_3\n",
      "print \"iV. Information content of fourth symbol (bits)\",Ix_4\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "i. Information content of first symbol (bits) =  1.0\n",
        "ii. Information content of second symbol (bits) =  2.0\n",
        "iii. Information content of third  symbol (bits) =  3.0\n",
        "iV. Information content of fourth symbol (bits) 3.0\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 2 - pg 488"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the amount of information\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "Px_i = 1./4#probability of a symbol\n",
      "\n",
      "#calculation\n",
      "Ix_i = (log(1/Px_i))/log(2)#formula for amount of information of a symbol\n",
      "\n",
      "#result\n",
      "print \"i. Amount of information (bits) = \",Ix_i\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "i. Amount of information (bits) =  2.0\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 3 - pg 489"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the amount of information\n",
      "#given\n",
      "import math\n",
      "from math import log\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "#since there are only two binary levels i.e. 1 or 0. Since, these two binary levels occur with equal             likelihood of occurrence will be\n",
      "Px_1 = 1./2#probability of zero level\n",
      "Px_2 = 1./2#probability of first level\n",
      "\n",
      "#calculations\n",
      "Ix_1 = log2(1/Px_1)#amount of information of zero level with base 2\n",
      "Ix_2 = log2(1/Px_2)#amount of information of first level with base 2\n",
      "Ix_1= log(1/Px_1)/log(2)#amount of information content with base 10\n",
      "Ix_2 = Ix_1 \n",
      "\n",
      "#result\n",
      "print \"i.Amount of information content wrt binary PCM 0 (bit) = \",Ix_1\n",
      "print \"ii.Amount of information content wrt binary PCM 1 (bit) = \",Ix_2\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "i.Amount of information content wrt binary PCM 0 (bit) =  1.0\n",
        "ii.Amount of information content wrt binary PCM 1 (bit) =  1.0\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 4 - pg 489"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the amount of information\n",
      "import math\n",
      "from math import log\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "#given\n",
      "Px_1 = 1./4#probability wrt to binary PCM '0'\n",
      "Px_2 = 3./4#probability wrt to binary PCM '1'\n",
      "\n",
      "#calculations\n",
      "Ix_1 = log2(1/Px_1)#amount of information of zero level with base 2\n",
      "Ix_2 = log2(1/Px_2)#amount of information of first level with base 2\n",
      "Ix_1= log(1/Px_1)/log(2)#amount of information content with base 10\n",
      "Ix_2= log(1/Px_2)/log(2)#amount of information content with base 10\n",
      "\n",
      "#results\n",
      "print \"i.Amount of information carried wrt to binary PCM 0 (bits) = \",Ix_1\n",
      "print \"ii.Amount of information carried wrt to binary PCM 1 (bits) = \",round(Ix_2,3)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "i.Amount of information carried wrt to binary PCM 0 (bits) =  2.0\n",
        "ii.Amount of information carried wrt to binary PCM 1 (bits) =  0.415\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 9 - pg 492"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the entropy and Amount of information\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "\n",
      "Px_1 = .4#probability of first symbol\n",
      "Px_2 = .3#probability of second symbol\n",
      "Px_3 = .2#probability of third symbol\n",
      "Px_4 = .1#probability of fourth symbol\n",
      "\n",
      "#calculations\n",
      "H_X = -Px_1*log2(Px_1)-Px_2*log2(Px_2)-Px_3*log2(Px_3)-Px_4*log2(Px_4);#entropy\n",
      "Px1x2x1x3 = Px_1*Px_2*Px_1*Px_3;#product of probabilities\n",
      "Ix1x2x1x3 =-log2(Px1x2x1x3);#information of four symbols\n",
      "Px4x3x3x2 = Px_4*Px_3*Px_3*Px_2;#product of probabilities\n",
      "Ix4x3x3x2 = -log2(Px4x3x3x2);#information of four symbols\n",
      "\n",
      "#results\n",
      "print \"i.Entropy (bits/symbol) = \",round(H_X,2)\n",
      "print \"ii.Amount of information contained in x1x2x1x3 (bits/symbol) = \",round(Ix1x2x1x3,2)\n",
      "print \"Thus,Ix1x2x1x3 < 7.4[=4*H_X]bits/symbol\"\n",
      "print \"iii.Amount of information contained in x4x3x3x2 (bits/symbol) = \",round(Ix4x3x3x2,2)\n",
      "print \"\\nThus we conclude that\\nIx4x3x3x2 > 7.4[=4*H_X]bits/symbol\"\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "i.Entropy (bits/symbol) =  1.85\n",
        "ii.Amount of information contained in x1x2x1x3 (bits/symbol) =  6.7\n",
        "Thus,Ix1x2x1x3 < 7.4[=4*H_X]bits/symbol\n",
        "iii.Amount of information contained in x4x3x3x2 (bits/symbol) =  9.7\n",
        "\n",
        "Thus we conclude that\n",
        "Ix4x3x3x2 > 7.4[=4*H_X]bits/symbol\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 12 - pg 495"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the average rate of information convyed\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "\n",
      "n = 2.*10**6#elements od black and white TV picture\n",
      "m = 16.#brightness levels of black and white TV picture\n",
      "o = 32.#repeated rate of pictures per second \n",
      "\n",
      "#calculations\n",
      "Px_i = 1./m#probability of  brightness levels of picture\n",
      "H_X = 0;\n",
      "for  i in range (0,15):\n",
      "\tH_Xi = (-1./(1./Px_i))*log2(1./(1./Px_i));\n",
      "\tH_X = H_X +H_Xi\n",
      "\n",
      "r = n*o#rate of symbols generated \n",
      "R = r*math.ceil(H_X)#average rate of information convyed \n",
      " \n",
      "#results\n",
      "print \"i. Average rate of information convyed (Mbits/seconds) = \",R/10**6\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "i. Average rate of information convyed (Mbits/seconds) =  256.0\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 13 - pg 495"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the average information rate\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "\n",
      "t_dot = .2#duration of dot symbol\n",
      "t_dash = .6#duration of dash symbol\n",
      "t_space = .2#time between the symbols\n",
      "#wkt sum of the probability is 1 i.e P_dot + P_dash = 1 hence\n",
      "#P_dot = 2*P_dash weget \n",
      "P_dot = 2./3#probality of dot symbol\n",
      "P_dash = 1./3#probality of dash symbol\n",
      "\n",
      "#calculations \n",
      "H_X = -P_dot*log2(P_dot)-P_dash*log2(P_dash);#entropy\n",
      "T_s = P_dot*t_dot + P_dash*t_dash +t_space;#average time per symbol\n",
      "r = 1/T_s;#average symbol rate \n",
      "R = r*H_X;#average information rate of the telegraph sourece\n",
      "\n",
      "#result\n",
      "print \"i.The average information rate of the telegraph source (bits/seconds) = \",round(R,3)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "i.The average information rate of the telegraph source (bits/seconds) =  1.722\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 14 - pg 496"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the entropy and information rate\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "#given symbols are equally likely all the symbols the probabilities are same\n",
      "Px_1 = 1./8;#probability of first symbol\n",
      "Px_2 = 1./8;#probability of second symbol\n",
      "Px_3 = 3./8;#probability of third symbol\n",
      "Px_4 = 3./8;#probability of fourth symbol\n",
      "\n",
      "r = 2#average symbol rate from problem 11.14\n",
      "\n",
      "#calculaitons\n",
      "H_X = Px_1*log2(1/Px_1) + Px_2*log2(1/Px_2) + Px_3*log2(1/Px_3) + Px_4*log2(1/Px_4);#entropy\n",
      "R = H_X*r;#information rate\n",
      "\n",
      "#results\n",
      "print \"i.Entropy (bits/symbol) = \",round(H_X,1)\n",
      "print \"ii.The information rate of all symbols (f_m bits/seconds) = \", round(R,1)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "i.Entropy (bits/symbol) =  1.8\n",
        "ii.The information rate of all symbols (f_m bits/seconds) =  3.6\n"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 15 - pg 497"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the information rate\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "#given symbols are equally likely all the symbols the probabilities are same\n",
      "Px_1 = 1./4;#probability of first symbol\n",
      "Px_2 = 1./4;#probability of second symbol\n",
      "Px_3 = 1./4;#probability of third symbol\n",
      "Px_4 = 1./4;#probability of fourth symbol\n",
      "r = 2#average symbol rate from problem 11.14\n",
      "\n",
      "#calculaitons\n",
      "H_X = Px_1*log2(1/Px_1) + Px_2*log2(1/Px_2) + Px_3*log2(1/Px_3) + Px_4*log2(1/Px_4);#entropy\n",
      "R = H_X*r;#information rate\n",
      "\n",
      "#results\n",
      "print \"i.The information rate of all symbols (f_m bits/seconds) = \", R\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "i.The information rate of all symbols (f_m bits/seconds) =  4.0\n"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 16 - pg 498"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the Entropy and rate of information\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "Px_1 = 1./2;#probability of first symbol\n",
      "Px_2 = 1./4;#probability of second symbol\n",
      "Px_3 = 1./8;#probability of third symbol\n",
      "Px_4 = 1./16;#probability of fourth symbol\n",
      "Px_4 = 1./16;#probability of fifth symbol\n",
      "T_b = 1*10**-3#time required for emittion of each symbol\n",
      "r = 1./(T_b)#symbol rate\n",
      "\n",
      "#calculations\n",
      "H_X = Px_1*log2(1/Px_1) + Px_2*log2(1/Px_2) + Px_3*log2(1/Px_3) + Px_4*log2(1/Px_4) + Px_4*log2(1/Px_4);\n",
      "R = r*H_X;#information rate\n",
      "\n",
      "#results\n",
      "print \"i.Entropy of five symbols (bits/symbol) = \",H_X\n",
      "print \"ii.Rate of information (bits/sec) = \",R\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "i.Entropy of five symbols (bits/symbol) =  1.875\n",
        "ii.Rate of information (bits/sec) =  1875.0\n"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 17 - pg 498"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the rate of information\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "Px_1 = 1./2;#probability of first symbol\n",
      "Px_2 = 1./4;#probability of second symbol\n",
      "Px_3 = 1./8;#probability of third symbol\n",
      "Px_4 = 1./16;#probability of fourth symbol\n",
      "Px_5 = 1./16;#probability of fifth symbol\n",
      "r = 16#outcomes per second\n",
      "\n",
      "#calculations\n",
      "H_X = Px_1*log2(1/Px_1) + Px_2*log2(1/Px_2) + Px_3*log2(1/Px_3) + Px_4*log2(1/Px_4) + Px_5*log2(1/Px_5);\n",
      "R = r*H_X;#information rate\n",
      "\n",
      "#result\n",
      "print \"Rate of information (bits/sec) = \",R\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Rate of information (bits/sec) =  30.0\n"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 18 - pg 499"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the rate of information\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "Px_1 = 1./4;#probability of first symbol\n",
      "Px_2 = 1./5;#probability of second symbol\n",
      "Px_3 = 1./5;#probability of third symbol\n",
      "Px_4 = 1./10;#probability of fourth symbol\n",
      "Px_5 = 1./10;#probability of fifth symbol\n",
      "Px_6 = 1./20;#probability of sixth symbol\n",
      "Px_7 = 1./20;#probability of seventh symbol\n",
      "Px_8 = 1./20;#probability of eigith symbol\n",
      "f_m = 10*10**3#freuency of tranamitting symbol\n",
      "\n",
      "#calculations\n",
      "H_X = Px_1*log2(1/Px_1) + Px_2*log2(1/Px_2) + Px_3*log2(1/Px_3) + Px_4*log2(1/Px_4) + Px_5*log2(1/Px_5) + Px_6*log2(1/Px_6)+ Px_7*log2(1/Px_7)+ Px_8*log2(1/Px_8);#entropy\n",
      "f_s = 2*f_m#sampling frequency\n",
      "r = f_s#sampling frequency equal to rate of transmission\n",
      "R = r*H_X;#information rate\n",
      "\n",
      "#result\n",
      "print \"Rate of information (bits/sec) = \",round(R,0)\n",
      "print \"Note:Their mistake in calculation of H_X in textbook\"\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Rate of information (bits/sec) =  54829.0\n",
        "Note:Their mistake in calculation of H_X in textbook\n"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 19 - pg 502"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the channel matrix\n",
      "import numpy\n",
      "#given\n",
      "#from fig\n",
      "P_X = numpy.matrix([.5, .5])#x matrix\n",
      "P_Xd = numpy.matrix([[.5, 0],[0, .5]])#diagonal x matrix\n",
      "#calculations\n",
      "P_YX = numpy.matrix([[.9, .1],[.2, .8]]);#yx matrix representation of given fig\n",
      "P_Y = P_X*P_YX#y matrix\n",
      "P_XY = P_Xd * P_YX#xy  matrix\n",
      "\n",
      "#results\n",
      "print \"i.Channel matrix of the channelP_YX \"\n",
      "print(P_YX)\n",
      "print \"ii.a.P(y1) =\",P_Y[0,0]\n",
      "print \"   b.P(y2) = \",P_Y[0,1]\n",
      "print \"iii.a.P(x1,y2) = \",P_XY[0,1]\n",
      "print \"     b.P(x2,y1) = \",P_XY[1,0]\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "i.Channel matrix of the channelP_YX \n",
        "[[ 0.9  0.1]\n",
        " [ 0.2  0.8]]\n",
        "ii.a.P(y1) = 0.55\n",
        "   b.P(y2) =  0.45\n",
        "iii.a.P(x1,y2) =  0.05\n",
        "     b.P(x2,y1) =  0.1\n"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 20 - pg 503"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the Channel matrix\n",
      "import numpy\n",
      "#given\n",
      "P_X = numpy.matrix([.5, .5])#x matrix\n",
      "#calculations\n",
      "P_YX = numpy.matrix([[.9, .1],[.2, .8]]);#yx matrix representation of given fig\\\n",
      "P_ZY = numpy.matrix([[.9, .1],[.2, .8]]);#yx matrix representation of given fig\n",
      "P_Y = P_X *P_YX#y matrix\n",
      "P_ZX = P_YX * P_ZY#zx  matrix\n",
      "P_Z = P_X *P_ZX#z matrix\n",
      "\n",
      "\n",
      "#results\n",
      "print \"i.Channel matrix of the channelP_ZX \"\n",
      "print(P_ZX)\n",
      "print(\"Matrix P(Z)\")\n",
      "print(P_Z)\n",
      "print \"a.P(Z1) = \",P_Z[0,0]\n",
      "print \"b.P(Z2) = \",P_Z[0,1]\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "i.Channel matrix of the channelP_ZX \n",
        "[[ 0.83  0.17]\n",
        " [ 0.34  0.66]]\n",
        "Matrix P(Z)\n",
        "[[ 0.585  0.415]]\n",
        "a.P(Z1) =  0.585\n",
        "b.P(Z2) =  0.415\n"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 21 - pg 504"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the probability associated with channel outputs\n",
      "#given\n",
      "import numpy\n",
      "P_X = numpy.matrix([.5, .5])#x matrix\n",
      "P_YX = numpy.matrix([[.8, .2, 0], [0, .2, .8]]);#yx matrix representation of given fig\n",
      "\n",
      "#calculations\n",
      "P_Y = P_X*P_YX;\n",
      "\n",
      "#results\n",
      "print \"probability associated with the channel outputs for p=.2 is\"\n",
      "print(P_Y)\n",
      "print \"a.P(Y1) = \",P_Y[0,0]\n",
      "print \"b.P(Y2) = \",P_Y[0,1]\n",
      "print \"C.P(Y3) = \",P_Y[0,2]\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "probability associated with the channel outputs for p=.2 is\n",
        "[[ 0.4  0.2  0.4]]\n",
        "a.P(Y1) =  0.4\n",
        "b.P(Y2) =  0.2\n",
        "C.P(Y3) =  0.4\n"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 28 - pg 504"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the I_XY value\n",
      "import numpy\n",
      "import math\n",
      "from math import log\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "#given\n",
      "#wkt P_Y = P_X*P_YX from previous problems\n",
      "alfa = .5\n",
      "P_1 = .1#probability for first case\n",
      "P_2 = .5#probability for second case\n",
      "\n",
      "#calculations\n",
      "P_X = numpy.matrix([alfa, alfa]);\n",
      "#first case\n",
      "P_YX = ([[1-P_1, P_1],[P_1, 1-P_1]]);\n",
      "P_Y1 = P_X*P_YX;\n",
      "H_Y1 = -P_Y1[0,0]*log2(P_Y1[0,0])-P_Y1[0,1]*log2(P_Y1[0,1]);\n",
      "Q_1 = P_1*log2(P_1)  + (1-P_1)*log2(1-P_1)#from proof\n",
      "I_XY1 =  1 + Q_1;\n",
      "#second case\n",
      "P_YX = ([[1-P_2, P_2],[P_2, 1-P_2]]);\n",
      "P_Y2 = P_X*P_YX;\n",
      "H_Y2 = -P_Y2[0,0]*log2(P_Y2[0,0])-P_Y2[0,1]*log2(P_Y2[0,1]);\n",
      "Q_2 = P_2*log2(P_2)  + (1-P_2)*log2(1-P_2)#from proof \n",
      "I_XY2 =  1 + Q_2;\n",
      "\n",
      "#results\n",
      "print \"I_XY for the first case = \",round(I_XY1,3)\n",
      "print \"I_XY for the second case = \",round(I_XY2,2)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "I_XY for the first case =  0.531\n",
        "I_XY for the second case =  0.0\n"
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 32 - pg 518"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the Entropy in all cases\n",
      "import math\n",
      "from math import log\n",
      "\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "#given\n",
      "a1 = 1.\n",
      "a2 = 2.\n",
      "a3 = .5\n",
      "\n",
      "#calculations\n",
      "H_X1 = log2(a1);#Entropy for first case\n",
      "H_X2 = log2(a2);#Entropy for second case\n",
      "H_X3 = log2(a3);#Entropy for third case\n",
      "\n",
      "#results\n",
      "print \"i.Entropy for first case =  \",H_X1\n",
      "print \"ii.Entropy for second case = \",H_X2\n",
      "print \"iii.Entropy for third case = \",H_X3\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "i.Entropy for first case =   0.0\n",
        "ii.Entropy for second case =  1.0\n",
        "iii.Entropy for third case =  -1.0\n"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 35 - pg 520"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the capacity of channel\n",
      "\n",
      "import math\n",
      "from math import log\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "#given\n",
      "B = 4000.#bandwidth of AWGN channel\n",
      "S = .1*10**-3#power of signal\n",
      "neta = 2*10**-12#spectral dencity \n",
      "N = neta*B;#power \n",
      "\n",
      "#calculations \n",
      "C = B * log2(1 + (S/N));#capacity of channel\n",
      "\n",
      "#result\n",
      "print \"Capacity of channel (b/s) = \",round(C,0)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Capacity of channel (b/s) =  54439.0\n"
       ]
      }
     ],
     "prompt_number": 19
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 36 - pg 521"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the information rate, bandwidth and S/N ratio\n",
      "import math\n",
      "from math import log\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "#given\n",
      "fm=4000. #Hz\n",
      "tim=1.25\n",
      "Pi=1/256.\n",
      "SN=100.\n",
      "#calculations \n",
      "fs=2*fm\n",
      "r=fs*tim\n",
      "H=log2(1/Pi)\n",
      "R=r*H\n",
      "C = r* log2(1 + SN);#capacity of channel\n",
      "SN2=(1/Pi-1)\n",
      "B2=R/log2(1+SN)\n",
      "#result\n",
      "print \"information rate of source (kb/s) = \",R/1000.\n",
      "print \"Capacity of channel (b/s) = \",round(C,0)\n",
      "print \"Final S/N ratio = \",SN2\n",
      "print \"bandwidth required  (kHz) = \",round(B2/1000.,0)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "information rate of source (kb/s) =  80.0\n",
        "Capacity of channel (b/s) =  66582.0\n",
        "Final S/N ratio =  255.0\n",
        "bandwidth required  (kHz) =  12.0\n"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 37 - pg 524"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the efficiency of code and code redundancy\n",
      "\n",
      "import math\n",
      "from math import log\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "#given\n",
      "Px_1 = 0.9#probability of first symbol\n",
      "Px_2 = 0.1#probability of second symbol\n",
      "n1 = 1.#length of the code for x_1\n",
      "n2 =1.#length of code for x_2\n",
      "\n",
      "#calculations\n",
      "#we know that the average code length L per symbol\n",
      "L = Px_1*n1 + Px_2*n2#code length\n",
      "H_X = -Px_1*log2(Px_1) - Px_2*log2(Px_2) #entropy\n",
      "neta = H_X/L#efficiency \n",
      "neta1 = neta*100#neta in percentage\n",
      "gama = 1 - neta#redundancy\n",
      "gama1 = gama*100#gama in percentage\n",
      "\n",
      "#results\n",
      "print \"i.Efficiency of code (percent) = \",round(neta1,1)\n",
      "print \"ii.Code redundancy (percent) =  \",round(gama1,1)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "i.Efficiency of code (percent) =  46.9\n",
        "ii.Code redundancy (percent) =   53.1\n"
       ]
      }
     ],
     "prompt_number": 21
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 38 - pg 524"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the efficiency of code and code redundancy\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "Px_1 = 0.81#probability of first symbol\n",
      "Px_2 = .09#probability of second symbol\n",
      "Px_3 = .09#probability of third symbol\n",
      "Px_4 = 0.01#probability of forth symbol\n",
      "n1 = 1.#length of code for a_1\n",
      "n2 =2.#length of code for a_2\n",
      "n3 = 3.#length of code for a_3\n",
      "n4 = 3.#length of code for a_4\n",
      "\n",
      "#calculations\n",
      "#we know that the average code length L per symbol\n",
      "L = Px_1*n1 + Px_2*n2 + Px_3*n3 + Px_4*n4 #code length\n",
      "H_X = -Px_1*log2(Px_1) - Px_2*log2(Px_2) - Px_3*log2(Px_3) - Px_4*log2(Px_4)#entropy \n",
      "neta = H_X/L#efficiency \n",
      "neta1 = neta*100#neta in percentage\n",
      "gama = 1 - neta#redundancy\n",
      "gama1 = gama*100#gama in percentage\n",
      "\n",
      "#results\n",
      "print \"i.Efficiency of code (percent) = \",round(neta1,1)\n",
      "print \"ii.Code redundancy (percent) =  \",round(gama1,1)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "i.Efficiency of code (percent) =  72.7\n",
        "ii.Code redundancy (percent) =   27.3\n"
       ]
      }
     ],
     "prompt_number": 22
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 44 - pg 529"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the efficiency \n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "P_x1 = 1./2#probability of first symbol\n",
      "P_x2 = 1./4#probability of second symbol\n",
      "P_x3 = 1./8#probability of third symbol\n",
      "P_x4 = 1./8#probability of fouth symbol\n",
      "n1 = 1.\n",
      "n2 = 2.\n",
      "n3 = 3.\n",
      "n4 = 3.\n",
      "\n",
      "#calculations\n",
      "I_x1 = -log2(P_x1);\n",
      "I_x2 = -log2(P_x2);\n",
      "I_x3 = -log2(P_x3);\n",
      "I_x4 = -log2(P_x4);\n",
      "H_x = P_x1*I_x1 +  P_x2*I_x2 +  P_x3*I_x3 +  P_x4*I_x4;\n",
      "L = P_x1*n1 + P_x2*n2 + P_x3*n3 + P_x4*n4;\n",
      "neta = H_x/L;\n",
      "P_neta = neta*100#efficiency in percentage\n",
      "\n",
      "#results\n",
      "print \"Efficiency = \",neta\n",
      "print \"Efficiency in percentage (percent) = \",P_neta\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Efficiency =  1.0\n",
        "Efficiency in percentage (percent) =  100.0\n"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 45 - pg 532"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the efficiency using shannon code and huffman code\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "P_x1 = .2#probability of first signal\n",
      "P_x2 = .2#probability of second signal\n",
      "P_x3 = .2#probability of third signal\n",
      "P_x4 = .2#probability of fourth signal\n",
      "P_x5 = .2#probability of fifth signal\n",
      "n1 = 2.#number of bits in code obtained from table given textbook\n",
      "n2 = 2.#number of bits in code obtained from table given textbook\n",
      "n3 = 2.#number of bits in code obtained from table given textbook\n",
      "n4 = 3.#number of bits in code obtained from table given textbook\n",
      "n5 = 3.#number of bits in code obtained from table given textbook\n",
      "\n",
      "#calculations\n",
      "I_x1 = -log2(P_x1);\n",
      "I_x2 = -log2(P_x2);\n",
      "I_x3 = -log2(P_x3);\n",
      "I_x4 = -log2(P_x4);\n",
      "I_x5 = -log2(P_x5);\n",
      "H_x = P_x1*I_x1 +  P_x2*I_x2 +  P_x3*I_x3 +  P_x4*I_x4 +  P_x5*I_x5;#entropy\n",
      "L1 = P_x1*n1 + P_x2*n2 + P_x3*n3 + P_x4*n4 + P_x5*n5;\n",
      "neta1 = H_x/L1;\n",
      "P_neta1 = neta1*100#efficiency in percentage using Shannon Fano code\n",
      "L2 = P_x1*n1 + P_x2*n2 + P_x3*n3 +P_x4*n4 +P_x5*n5\n",
      "neta2 = H_x/L2;\n",
      "P_neta2 = neta2*100#efficiency in percentage using huffman code\n",
      "\n",
      "#results\n",
      "print \"Efficiency in percentage using Shannon Fano code (percent) = \",round(P_neta1,1)\n",
      "print \"Efficiency in percentage using huffman code (percent) = \",round(P_neta2,1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Efficiency in percentage using Shannon Fano code (percent) =  96.7\n",
        "Efficiency in percentage using huffman code (percent) =  96.7\n"
       ]
      }
     ],
     "prompt_number": 24
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 46 - pg 532"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the efficiency using shannon code and huffman code\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "P_x1 = .4#probability of first signal\n",
      "P_x2 = .19#probability of second signal\n",
      "P_x3 = .16#probability of third signal\n",
      "P_x4 = .15#probability of fourth signal\n",
      "P_x5 = .1#probability of fifth signal\n",
      "n1 = 1.#number of bits in code obtained from table given textbook\n",
      "n2 = 2.#number of bits in code obtained from table given textbook\n",
      "n3 = 2.#number of bits in code obtained from table given textbook\n",
      "n4 = 3.#number of bits in code obtained from table given textbook\n",
      "n5 = 3.#number of bits in code obtained from table given textbook\n",
      "\n",
      "#calculations\n",
      "I_x1 = -log2(P_x1);\n",
      "I_x2 = -log2(P_x2);\n",
      "I_x3 = -log2(P_x3);\n",
      "I_x4 = -log2(P_x4);\n",
      "I_x5 = -log2(P_x5);\n",
      "H_x = P_x1*I_x1 +  P_x2*I_x2 +  P_x3*I_x3 +  P_x4*I_x4 +  P_x5*I_x5;#entropy\n",
      "L1 = P_x1*n1 + P_x2*n2 + P_x3*n3 + P_x4*n4 + P_x5*n5;\n",
      "neta1 = H_x/L1;\n",
      "P_neta1 = neta1*100#efficiency in percentage using Shannon Fano code\n",
      "L2 = P_x1*1 + (P_x2 + P_x3 +P_x4 +P_x5 )*3\n",
      "neta2 = H_x/L2;\n",
      "P_neta2 = neta2*100#efficiency in percentage using huffman code\n",
      "\n",
      "#results\n",
      "print \"Efficiency in percentage using Shannon Fano code (percent) = \",round(P_neta1,1)\n",
      "print \"Efficiency in percentage using huffman code (percent) = \",round(P_neta2,1)\n",
      "print \"Note: There is mistake in the textbook in calculation of L using SHannon Fano code\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Efficiency in percentage using Shannon Fano code (percent) =  116.2\n",
        "Efficiency in percentage using huffman code (percent) =  97.7\n",
        "Note: There is mistake in the textbook in calculation of L using SHannon Fano code\n"
       ]
      }
     ],
     "prompt_number": 26
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 47 - pg 532"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the Efficiency\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "P_x1 = .05#probability of first signal\n",
      "P_x2 = .15#probability of second signal\n",
      "P_x3 = .2#probability of third signal\n",
      "P_x4 = .05#probability of fourth signal\n",
      "P_x5 = .15#probability of fifth signal\n",
      "P_x6 = .3#probability of sixth signal\n",
      "P_x7 = .1#probability of seventh signal\n",
      "n1 = 4.#number of bits in code obtained from table given textbook\n",
      "n2 = 3.#number of bits in code obtained from table given textbook\n",
      "n3 = 2.#number of bits in code obtained from table given textbook\n",
      "n4 = 4.#number of bits in code obtained from table given textbook\n",
      "n5 = 3.#number of bits in code obtained from table given textbook\n",
      "n6 = 2.#number of bits in code obtained from table given textbook\n",
      "n7 = 3.#number of bits in code obtained from table given textbook\n",
      "\n",
      "#calculations\n",
      "I_x1 = -log2(P_x1);\n",
      "I_x2 = -log2(P_x2);\n",
      "I_x3 = -log2(P_x3);\n",
      "I_x4 = -log2(P_x4);\n",
      "I_x5 = -log2(P_x5);\n",
      "I_x6 = -log2(P_x6);\n",
      "I_x7 = -log2(P_x7);\n",
      "H_x = P_x1*I_x1 +  P_x2*I_x2 +  P_x3*I_x3 +  P_x4*I_x4 +  P_x5*I_x5 + P_x6*I_x6 + P_x7*I_x7;#entropy\n",
      "L = P_x1*n1 + P_x2*n2 + P_x3*n3 + P_x4*n4 + P_x5*n5 + P_x6*n6 + P_x7*n7;\n",
      "neta = (H_x*100)/L#Efficiency in percentage\n",
      "\n",
      "#results\n",
      "print \"Efficiency in percentage (percent) = \",round(neta,2)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Efficiency in percentage (percent) =  98.88\n"
       ]
      }
     ],
     "prompt_number": 27
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 49 - pg 534"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the Variance of codeword length\n",
      "\n",
      "#given\n",
      "P_x1 = .4#probability of first signal\n",
      "P_x2 = .2#probability of second signal\n",
      "P_x3 = .8#probability of third signal\n",
      "P_x4 = .08#probability of fourth signal\n",
      "P_x5 = .02#probability of fifth signal\n",
      "n1 = 2.#number of bits in code obtained from table given textbook\n",
      "n2 = 3.#number of bits in code obtained from table given textbook\n",
      "n3 = 1.#number of bits in code obtained from table given textbook\n",
      "n4 = 4.#number of bits in code obtained from table given textbook\n",
      "n5 = 4.#number of bits in code obtained from table given textbook\n",
      "\n",
      "#calculations\n",
      "L = P_x1*n1 + P_x2*n2 + P_x3*n3 + P_x4*n4 + P_x5*n5;#average codeword length per symbol\n",
      "#since sigma = sqrt(summation of product of probability and (n- L)**2)\n",
      "sigmasquare = P_x1*(n1-L)**2 + P_x2*(n2-L)**2 +P_x3*(n3-L)**2 + P_x4*(n4-L)**2 +P_x5*(n5-L)**2;#Variance of codewoed length\n",
      "\n",
      "#results\n",
      "print \"Variance of codeword length = \",sigmasquare\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Variance of codeword length =  2.42\n"
       ]
      }
     ],
     "prompt_number": 28
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 50 - pg 535"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the Entropy and Information rate\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "P_x1 = 1./2#probability of first signal\n",
      "P_x2 = 1./4#probability of second signal\n",
      "P_x3 = 1./8#probability of third signal\n",
      "P_x4 = 1./16#probability of fourth signal\n",
      "P_x5 = 1./32#probability of fifth signal\n",
      "P_x6 = 1./32#probability of sixth signal\n",
      "r = 16#message rate in outcomes per second\n",
      "\n",
      "#calculations\n",
      "I_x1 = -log2(P_x1);\n",
      "I_x2 = -log2(P_x2);\n",
      "I_x3 = -log2(P_x3);\n",
      "I_x4 = -log2(P_x4);\n",
      "I_x5 = -log2(P_x5);\n",
      "I_x6 = -log2(P_x6);\n",
      "H_X = P_x1*I_x1 +  P_x2*I_x2 +  P_x3*I_x3 +  P_x4*I_x4 +  P_x5*I_x5 + P_x6*I_x6 #entropy\n",
      "R = H_X*r#Information rate\n",
      "\n",
      "#results\n",
      "print \"Entropy of the system (bits/message) = \",H_X\n",
      "print \"Information rate (bits/seconds) = \",R\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Entropy of the system (bits/message) =  1.9375\n",
        "Information rate (bits/seconds) =  31.0\n"
       ]
      }
     ],
     "prompt_number": 29
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 51 - pg 535"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the Entropy\n",
      "import numpy\n",
      "#given\n",
      "import math\n",
      "from math import log\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "P_x1 = .3#probability of first signal\n",
      "P_x2 = .4#probability of second signal\n",
      "P_x3 = .3#probability of third signal\n",
      "P_YX = numpy.matrix([[.8, .2, 0],[0, .1, 0],[0, .3, 0.7]])#matrix obtained from the figure \n",
      "\n",
      "#calculations\n",
      "I_x1 = -log2(P_x1);\n",
      "I_x2 = -log2(P_x2);\n",
      "I_x3 = -log2(P_x3);\n",
      "H_X = P_x1*I_x1 +  P_x2*I_x2 +  P_x3*I_x3 #entropy\n",
      "P_y1 = P_YX[0,0]*P_x1 + P_YX[0,1]*P_x1 + P_YX[0,2]*P_x1;\n",
      "P_y2 = P_YX[1,0]*P_x2 + P_YX[1,1]*P_x2 + P_YX[1,2]*P_x2;\n",
      "P_y3 = P_YX[2,0]*P_x3 + P_YX[2,1]*P_x3 + P_YX[2,2]*P_x3;\n",
      "I_y1 = -log2(P_y1);\n",
      "I_y2 = -log2(P_y2);\n",
      "I_y3 = -log2(P_y3);\n",
      "H_Y = -P_y1*I_y1 -  P_y2*I_y2 -  P_y3*I_y3 #entropy\n",
      "\n",
      "#results\n",
      "print \" Entropy H(X) = \",round(H_X,3)\n",
      "print \"Entropy H(Y) = \",round(H_Y,3)\n",
      "print \" Note:There is mistake in the calculation of P_y3 in the textbook so their is change in entropy H_Y\"\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " Entropy H(X) =  1.571\n",
        "Entropy H(Y) =  -1.228\n",
        " Note:There is mistake in the calculation of P_y3 in the textbook so their is change in entropy H_Y\n"
       ]
      }
     ],
     "prompt_number": 30
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 52 - pg 536"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the Entropy of the second order extension\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "\n",
      "P_x1 = .7#probability of first signal\n",
      "P_x2 = .15#probability of second signal\n",
      "P_x3 = .15#probability of third signal\n",
      "n = 2#second order extention\n",
      "\n",
      "#calculations\n",
      "I_x1 = -log2(P_x1);\n",
      "I_x2 = -log2(P_x2);\n",
      "I_x3 = -log2(P_x3);\n",
      "H_x = P_x1*I_x1 +  P_x2*I_x2 +  P_x3*I_x3#entropy\n",
      "H_x2 = n*H_x#entropy of second order extention\n",
      "\n",
      "#results\n",
      "print \"Entropy of second order extension (bits/symbol) = \",round(H_x2,3)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Entropy of second order extension (bits/symbol) =  2.363\n"
       ]
      }
     ],
     "prompt_number": 31
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 54 - pg 537"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the Entropy of the source\n",
      "import math\n",
      "from math import log\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "\n",
      "#given\n",
      "P_x1 = 1./3#probability of first signal\n",
      "P_x2 = 1./6#probability of second signal\n",
      "P_x3 = 1./4#probability of third signal\n",
      "P_x4 = 1./4#probability of fourth signal\n",
      "\n",
      "#calculations\n",
      "I_x1 = -log2(P_x1);\n",
      "I_x2 = -log2(P_x2);\n",
      "I_x3 = -log2(P_x3);\n",
      "I_x4 = -log2(P_x4);\n",
      "H_x = P_x1*I_x1 +  P_x2*I_x2 +  P_x3*I_x3 +  P_x4*I_x4 #entropy\n",
      "\n",
      "#results\n",
      "print \"Entropy of the source (bits/symbol) =  \",round(H_x,4)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Entropy of the source (bits/symbol) =   1.9591\n"
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 55 - pg 538"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the Average number of bits per message\n",
      "\n",
      "#given\n",
      "P_x1 = 1./2#probability of first signal\n",
      "P_x2 = 1./4#probability of second signal\n",
      "P_x3 = 1./8#probability of third signal\n",
      "P_x4 = 1./16#probability of fourth signal\n",
      "P_x5 = 1./16#probability of fifth signal\n",
      "n1 = 1.#number of bits in code obtained from table given textbook\n",
      "n2 = 2.#number of bits in code obtained from table given textbook\n",
      "n3 = 3.#number of bits in code obtained from table given textbook\n",
      "n4 = 4.#number of bits in code obtained from table given textbook\n",
      "n5 = 4.#number of bits in code obtained from table given textbook\n",
      "\n",
      "#calculations\n",
      "L = P_x1*n1 + P_x2*n2 + P_x3*n3 + P_x4*n4 + P_x5*n5;#Average number of bits per message\n",
      "\n",
      "#results\n",
      "print \"Average number of bits per message (bits) = \",L\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Average number of bits per message (bits) =  1.875\n"
       ]
      }
     ],
     "prompt_number": 33
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Example 56 - pg 538"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#calculate the Information capacity of telephone channel\n",
      "import math\n",
      "from math import log,exp\n",
      "#given\n",
      "def log2(x):\n",
      "\ty=log(x)/log(2)\n",
      " \treturn y\n",
      "B = 3.4*10**3#bandwidth\n",
      "SbyN = 30#signal to the noise ratio in dB\n",
      "\n",
      "\n",
      "#calculations\n",
      "SbyN1 = exp((SbyN/10)*log(10))#signal to noise ratio \n",
      "C = B*log2(1+SbyN1);\n",
      "\n",
      "#result\n",
      "print \"Information capacity of the telephone channel (kbps) = \",round(C/1000.,2)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Information capacity of the telephone channel (kbps) =  33.89\n"
       ]
      }
     ],
     "prompt_number": 34
    }
   ],
   "metadata": {}
  }
 ]
}