[
  {
    "path": ".gitignore",
    "content": "# Compiled Lua sources\nluac.out\n\n# Torch serialised objects\n*.t7\n\n# Images\n*.jpg\n\n# luarocks build files\n*.src.rock\n*.zip\n*.tar.gz\n\n# Object files\n*.o\n*.os\n*.ko\n*.obj\n*.elf\n\n# Precompiled Headers\n*.gch\n*.pch\n\n# Libraries\n*.lib\n*.a\n*.la\n*.lo\n*.def\n*.exp\n\n# Shared objects (inc. Windows DLLs)\n*.dll\n*.so\n*.so.*\n*.dylib\n\n# Executables\n*.exe\n*.out\n*.app\n*.i*86\n*.x86_64\n*.hex\n\n"
  },
  {
    "path": ".gitmodules",
    "content": "[submodule \"optimize-net\"]\n\tpath = optimize-net\n\turl = https://github.com/1adrianb/optimize-net\n[submodule \"bnn.torch\"]\n\tpath = bnn.torch\n\turl = https://github.com/1adrianb/bnn.torch\n"
  },
  {
    "path": "LICENCE",
    "content": "Copyright (c) 2017, University of Nottingham\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n1.Redistributions of source code must retain the above copyright notice, this\n  list of conditions and the following disclaimer.\n\n2.Redistributions in binary form must reproduce the above copyright notice,\n  this list of conditions and the following disclaimer in the documentation\n  and/or other materials provided with the distribution.\n\n3.Neither the name of the paper nor the names of its\n  contributors may be used to endorse or promote products derived from\n  this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n"
  },
  {
    "path": "README.md",
    "content": "# Binarized Convolutional Landmark Localizers for Human Pose Estimation and Face Alignment with Limited Resources \n\nThis code implements a demo of the Binarized Convolutional Landmark Localizers for Human Pose Estimation and Face Alignment with Limited Resources paper by Adrian Bulat and Georgios Tzimiropoulos.\n\n**[2021 Update]: PyTorch repo with training code for BNN available here: [https://github.com/1adrianb/binary-networks-pytorch](https://github.com/1adrianb/binary-networks-pytorch)**\n\n**For the Face Alignment demo please check: [https://github.com/1adrianb/binary-face-alignment](https://github.com/1adrianb/binary-face-alignment)**\n\n## Requirements\n- Install the latest [Torch7](http://torch.ch/docs/getting-started.html) version (for Windows, please follow the instructions avaialable [here](https://github.com/torch/distro/blob/master/win-files/README.md))\n\n### Packages\n- [cutorch](https://github.com/torch/cutorch)\n- [nn](https://github.com/torch/nn)\n- [cudnn](https://github.com/soumith/cudnn.torch) (cudnn5 preffered)\n- [xlua](https://github.com/torch/xlua)\n- [image](https://github.com/torch/image)\n- [gnuplot](https://github.com/torch/gnuplot)\n- [cURL](https://github.com/Lua-cURL/Lua-cURLv3)\n- [paths](https://github.com/torch/paths)\n\n## Setup\nClone the github repository\n```bash\ngit clone https://github.com/1adrianb/binary-human-pose-estimation --recursive\ncd binary-human-pose-estimation\n```\n\nBuild and install the BinaryConvolution package\n```bash\ncd bnn.torch/; luarocks make; cd ..;\n```\n\nInstall the modified optnet package\n```bash\ncd optimize-net/; luarocks make rocks/optnet-scm-1.rockspec; cd ..;\n```\n\nRun the following command to prepare the files required by the demo. This will download 10 images from the MPII dataset alongside the dataset structure converted to .t7\n```bash\nth download-content.lua\n```\nDownload the model available bellow and place it in the models folder. \n\n## Usage\n\nIn order to run the demo simply type:\n```bash\nth main.lua\n```\n\n## Pretrained models\n\n| Layer type | Model Size | MPII  error |\n| ------------- | ----------- | ----------- |\n| [MPII](https://www.adrianbulat.com/downloads/BinaryHumanPose/human_pose_binary.t7)        | 1.3MB |76.0        |\n\nNote: More pretrained models will be added soon\n\n## Notes\n\nFor more details/questions please visit the [project page](https://www.adrianbulat.com/binary-cnn-landmarks) or send an email at adrian.bulat@nottingham.ac.uk\n\n\n\n\n"
  },
  {
    "path": "download-content.lua",
    "content": "local cURL = require 'cURL'\nlocal paths = require 'paths'\n\n-- Create the directories if needed\nif not paths.dirp('dataset') then paths.mkdir('dataset') end\nif not paths.dirp('dataset/mpii/images') then paths.mkdir('dataset/mpii/images') end\n\n-- Url, location\nlocal fileList = {\n\t{'https://www.adrianbulat.com/downloads/ECCV16/mpii_dataset.t7', 'dataset/mpii_dataset.t7'},\n\t{'https://www.adrianbulat.com/downloads/BinaryHumanPose/images/005808361.jpg', 'dataset/mpii/images/005808361.jpg'},\n\t{'https://www.adrianbulat.com/downloads/BinaryHumanPose/images/072245212.jpg', 'dataset/mpii/images/072245212.jpg'},\n\t{'https://www.adrianbulat.com/downloads/BinaryHumanPose/images/060754485.jpg', 'dataset/mpii/images/060754485.jpg'},\n\t{'https://www.adrianbulat.com/downloads/BinaryHumanPose/images/053710654.jpg', 'dataset/mpii/images/053710654.jpg'},\n\t{'https://www.adrianbulat.com/downloads/BinaryHumanPose/images/051074730.jpg', 'dataset/mpii/images/051074730.jpg'},\n\t{'https://www.adrianbulat.com/downloads/BinaryHumanPose/images/033761517.jpg', 'dataset/mpii/images/033761517.jpg'},\n\t{'https://www.adrianbulat.com/downloads/BinaryHumanPose/images/031800347.jpg', 'dataset/mpii/images/031800347.jpg'},\n\t{'https://www.adrianbulat.com/downloads/BinaryHumanPose/images/023724909.jpg', 'dataset/mpii/images/023724909.jpg'},\n\t{'https://www.adrianbulat.com/downloads/BinaryHumanPose/images/072818876.jpg', 'dataset/mpii/images/072818876.jpg'},\n\t{'https://www.adrianbulat.com/downloads/BinaryHumanPose/images/061062004.jpg', 'dataset/mpii/images/061062004.jpg'},\n}\n\nlocal m = cURL.multi()\n\nfor i = 1, #fileList do\n\t-- Open files\n\tfileList[i][2] = io.open(fileList[i][2], \"w+b\")\n\n\t-- Add the url handles\n\tfileList[i][1] = cURL.easy{url = fileList[i][1], writefunction = fileList[i][2]}\n\tm:add_handle(fileList[i][1])\nend\n\nprint(\"Downloading files, please wait...\")\n-- Based on https://github.com/Lua-cURL/Lua-cURLv3/blob/master/examples/cURLv3/multi2.lua\nlocal remain = #fileList\nwhile remain > 0 do\n\tlocal last = m:perform()\n\tif last < remain then\n\t\twhile true do\n\t\t\tlocal e, ok, err = m:info_read(true)\n\t\t\tif e == 0 then break end -- no more finished tasks\n\t\t\tif ok then\n\t\t\t\tprint(e:getinfo_effective_url(), '-', '\\027[00;92mOK\\027[00m')\n\t\t\telse\n\t\t\t\tprint(e:getinfo_effective_url(), '-', '\\027[00;91mFail\\027[00m')\n\t\t\tend\n\t\t\te:close()\n\t\tend\n\tend \n\tremain = last\n\n\tm:wait() \nend\t\n"
  },
  {
    "path": "main.lua",
    "content": "require 'torch'\r\nrequire 'nn'\r\nrequire 'cudnn'\r\nrequire 'paths'\r\n\r\nrequire 'bnn'\r\nlocal optnet = require 'optnet'\r\n\r\nrequire 'gnuplot'\r\nrequire 'image'\r\nrequire 'xlua'\r\nlocal utils = require 'utils'\r\nlocal opts = require('opts')(arg)\r\n\r\ntorch.setheaptracking(true)\r\ntorch.setdefaulttensortype('torch.FloatTensor')\r\ntorch.setnumthreads(1)\r\n\r\nlocal model = torch.load('models/human_pose_binary.t7')\r\nmodel:evaluate()\r\n\r\nlocal fileLists = utils.getFileList(opts)\r\nlocal predictions = {}\r\nlocal output = torch.CudaTensor(1,16,64,64)\r\n\r\noptimize_opts = {inplace=true, reuseBuffers=true, mode='inference'}\r\noptnet.optimizeMemory(model, torch.zeros(1,3,256,256):cuda(), optimize_opts)\r\n\r\nif opts.mode == 'eval' then xlua.progress(0,#fileLists) end\r\nfor i = 1, #fileLists do\r\n\tfileLists[i].image = 'dataset/mpii/images/'..fileLists[i].image\r\n\t\r\n\tlocal img = image.load(fileLists[i].image)\r\n\tlocal originalSize = img:size()\r\n\r\n\timg = utils.crop(img, fileLists[i].center, fileLists[i].scale, 256)\r\n\timg = img:cuda():view(1,3,256,256)\r\n\t\r\n\toutput:copy(model:forward(img))\r\n\toutput:add(utils.flip(utils.shuffleLR(model:forward(utils.flip(img)))))\r\n\r\n\tlocal preds_hm, preds_img = utils.getPreds(output, fileLists[i].center, fileLists[i].scale)\r\n\t\r\n\tif opts.mode == 'demo' then\r\n\t\tutils.plot(fileLists[i].image,preds_img:view(16,2),torch.Tensor{originalSize[3],originalSize[2]})\r\n\t\tio.read() -- Wait for user input\r\n\tend\r\n\t\r\n\tif opts.mode == 'eval' then\r\n\t\tpredictions[i] = preds_img:clone()\r\n\t\txlua.progress(i, #fileLists)\r\n\tend\r\nend\r\n\r\nif opts.mode == 'demo' then gnuplot.closeall() end\r\n\r\nif opts.mode == 'eval' then\r\n\tpredictions = torch.cat(predictions,1)\r\n\tlocal dists = utils.calcDistance(predictions,fileLists)\r\n\tutils.calculateMetrics(dists)\r\nend"
  },
  {
    "path": "opts.lua",
    "content": "local function parse(arg)\r\n\tlocal cmd = torch.CmdLine()\r\n\tcmd:text()\r\n\tcmd:text('Binary Human Pose demo script')\r\n\tcmd:text('Please visit https://www.adrianbulat.com for additional details')\r\n\tcmd:text()\r\n\tcmd:text('Options:')\r\n\t\r\n\tcmd:option('-mode',\t\t\t'demo', 'Options: demo | eval')\r\n\t\r\n\tcmd:text()\r\n\t\r\n\tlocal opt = cmd:parse(arg or {})\r\n\t\r\n\treturn opt \r\nend\r\n\r\nreturn parse"
  },
  {
    "path": "utils.lua",
    "content": "local utils = {}\r\n\r\n-- Transform the coordinates from the original image space to the cropped one\r\nfunction utils.transform(pt, center, scale, res, invert)\r\n    -- Define the transformation matrix\r\n    local pt_new = torch.ones(3)\r\n    pt_new[1], pt_new[2] = pt[1], pt[2]\r\n    local h = 200*scale\r\n    local t = torch.eye(3)\r\n    t[1][1], t[2][2] = res/h, res/h\r\n    t[1][3], t[2][3] = res*(-center[1]/h+0.5), res*(-center[2]/h+0.5)\r\n    if invert then\r\n        t = torch.inverse(t)\r\n    end\r\n    local new_point = (t*pt_new):sub(1,2):int()\r\n    return new_point\r\nend\r\n\r\n-- Crop based on the image center & scale\r\nfunction utils.crop(img, center, scale, res)\r\n    local l1 = utils.transform({1,1}, center, scale, res, true)\r\n    local l2 = utils.transform({res,res}, center, scale, res, true)\r\n\r\n    local pad = math.floor(torch.norm((l1 - l2):float())/2 - (l2[1]-l1[1])/2)\r\n    \r\n    if img:nDimension() < 3 then\r\n      img = torch.repeatTensor(img,3,1,1)\r\n    end\r\n\r\n    local newDim = torch.IntTensor({img:size(1), l2[2] - l1[2], l2[1] - l1[1]})\r\n    local newImg = torch.zeros(newDim[1],newDim[2],newDim[3])\r\n    local height, width = img:size(2), img:size(3)\r\n\r\n    local newX = torch.Tensor({math.max(1, -l1[1]+1), math.min(l2[1], width) - l1[1]})\r\n    local newY = torch.Tensor({math.max(1, -l1[2]+1), math.min(l2[2], height) - l1[2]})\r\n    local oldX = torch.Tensor({math.max(1, l1[1]+1), math.min(l2[1], width)})\r\n    local oldY = torch.Tensor({math.max(1, l1[2]+1), math.min(l2[2], height)})\r\n\r\n    newImg:sub(1,newDim[1],newY[1],newY[2],newX[1],newX[2]):copy(img:sub(1,newDim[1],oldY[1],oldY[2],oldX[1],oldX[2]))\r\n\r\n    newImg = image.scale(newImg,res,res)\r\n    return newImg\r\nend\r\n\r\nfunction utils.getPreds(heatmaps, center, scale)\r\n    if heatmaps:nDimension() == 3 then heatmaps = heatmaps:view(1, unpack(heatmaps:size():totable())) end\r\n\r\n    -- Get locations of maximum activations\r\n    local max, idx = torch.max(heatmaps:view(heatmaps:size(1), heatmaps:size(2), heatmaps:size(3) * heatmaps:size(4)), 3)\r\n    local preds = torch.repeatTensor(idx, 1, 1, 2):float()\r\n    preds[{{}, {}, 1}]:apply(function(x) return (x - 1) % heatmaps:size(4) + 1 end)\r\n    preds[{{}, {}, 2}]:add(-1):div(heatmaps:size(3)):floor():add(1)\r\n\r\n    for i = 1,preds:size(1) do        \r\n        for j = 1,preds:size(2) do\r\n            local hm = heatmaps[{i,j,{}}]\r\n            local pX, pY = preds[{i,j,1}], preds[{i,j,2}]\r\n            if pX > 1 and pX < 64 and pY > 1 and pY < 64 then\r\n                local diff = torch.FloatTensor({hm[pY][pX+1]-hm[pY][pX-1], hm[pY+1][pX]-hm[pY-1][pX]})\r\n                preds[i][j]:add(diff:sign():mul(.25))\r\n            end\r\n        end\r\n    end\r\n    preds:add(-0.5)\r\n\r\n    -- Get the coordinates in the original space\r\n    local preds_orig = torch.zeros(preds:size())\r\n    for i = 1, heatmaps:size(1) do\r\n        for j = 1, heatmaps:size(2) do\r\n            preds_orig[i][j] = utils.transform(preds[i][j],center,scale,heatmaps:size(3),true)\r\n        end\r\n    end\r\n    return preds, preds_orig\r\nend\r\n\r\nfunction utils.shuffleLR(x)\r\n    local dim\r\n    if x:nDimension() == 4 then\r\n        dim = 2\r\n    else\r\n        assert(x:nDimension() == 3)\r\n        dim = 1\r\n    end\r\n\r\n    local matched_parts = {\r\n        {1,6},   {2,5},   {3,4},\r\n        {11,16}, {12,15}, {13,14}\r\n    }\r\n\r\n    for i = 1,#matched_parts do\r\n        local idx1, idx2 = unpack(matched_parts[i])\r\n        local tmp = x:narrow(dim, idx1, 1):clone()\r\n        x:narrow(dim, idx1, 1):copy(x:narrow(dim, idx2, 1))\r\n        x:narrow(dim, idx2, 1):copy(tmp)\r\n    end\r\n\r\n    return x\r\nend\r\n\r\nfunction utils.flip(x)\r\n    local y = torch.FloatTensor(x:size())\r\n    for i = 1, x:size(1) do\r\n        image.hflip(y[i], x[i]:float())\r\n    end\r\n    return y:typeAs(x)\r\nend\r\n\r\nfunction utils.calcDistance(predictions,groundTruth)\r\n  local n = predictions:size()[1]\r\n  gnds = torch.Tensor(n,16,2)\r\n  for i=1,n do\r\n    gnds[{{i},{},{}}] = groundTruth[i].points\r\n  end\r\n\r\n  local dists = torch.Tensor(predictions:size(2),predictions:size(1))\r\n  -- Calculate L2\r\n\tfor i = 1,predictions:size(1) do\r\n\t\tfor j = 1,predictions:size(2) do\r\n\t\t\tif gnds[i][j][1] > 1 and gnds[i][j][2] > 1 then\r\n\t\t\t\tdists[j][i] = torch.dist(gnds[i][j],predictions[i][j])/groundTruth[i].headSize\r\n\t\t\telse\r\n\t\t\t\tdists[j][i] = -1\r\n\t\t\tend\r\n\t\tend\r\n\tend\r\n\r\n  return dists\r\nend\r\n\r\nfunction utils.getFileList(opts)\r\n\tlocal fileLists = {}\r\n\ttempFileList = torch.load('dataset/mpii_dataset.t7')\r\n    if opts.mode == 'demo' then\r\n        local idxs = {1,5,16,17,18,24,28,63,66,104}\r\n        for i = 1, #idxs do\r\n            fileLists[i] = tempFileList[idxs[i]]\r\n        end\r\n\telse\r\n\t\tfor i = 1, #tempFileList do\r\n\t\t\tif tempFileList[i]['type'] == 0 then\r\n\t\t\t\tfileLists[#fileLists+1] = tempFileList[i]\r\n\t\t\tend\r\n\t\tend\r\n    end\r\n\treturn fileLists\r\nend\r\n\r\n-- Requires gnuplot\r\nfunction utils.plot(surface, points, size)\r\n\tpoints = points:view(16,2)\r\n   \r\n\tlocal matched_parts = {\r\n\t\t{1,2}, {2,3}, {3,7},\r\n\t\t{4,5}, {5,6}, {4,7},\r\n\t\t{9,10},{7,8},\r\n\t\t{11,12}, {12,13}, {13,8},\r\n\t\t{8,14}, {14,15}, {15,16}\r\n\t}\r\n\t\r\n\tlocal parts_colours = {\r\n\t\t\"blue\", \"blue\", \"blue\",\r\n\t\t\"red\", \"red\", \"red\",\r\n\t\t\"#9400D3\", \"#9400D3\",\r\n\t\t\"blue\", \"blue\", \"blue\",\r\n\t\t\"red\", \"red\", \"red\"\r\n\t}\r\n\t\r\n    gnuplot.figure(1)\r\n    gnuplot.raw(\"set size ratio -1\")\r\n\tgnuplot.raw(\"set xrange [0:\"..size[1]..\"]\")\r\n\tgnuplot.raw(\"set yrange [0:\"..size[2]..\"]\")\r\n    gnuplot.raw(\"unset key; unset tics; unset border;\")\r\n\tgnuplot.raw(\"set multiplot layout 1,1 margins 0.05,0.95,.1,.99 spacing 0,0\")\r\n    gnuplot.raw(\"plot '\"..surface..\"' binary filetype=jpg with rgbimage\")  \r\n\r\n\tgnuplot.raw(\" set yrange [\"..size[2]..\":0] \") \r\n\r\n\tcommands = {}\r\n\tfor i = 1, #matched_parts do\r\n\t\tcommands[i] = {torch.Tensor{points[matched_parts[i][1]][1],points[matched_parts[i][2]][1]},torch.Tensor{points[matched_parts[i][1]][2],points[matched_parts[i][2]][2]},'with lines lw 5 linecolor rgb \"'..parts_colours[i]..'\"'}\r\n\tend\r\n\tgnuplot.plot(unpack(commands))\r\n\tgnuplot.raw(\"unset multiplot\")\r\nend\r\n\r\nlocal function displayPCKh(dists, idxs, title, disp_key)\r\n\tlocal xs = torch.linspace(0,0.5,30)\r\n\tlocal ys = torch.zeros(xs:size(1))\r\n\tlocal total = {dists[{idxs[1],{}}]:gt(-1):sum(),\r\n\t\t\t\t\tdists[{idxs[2],{}}]:gt(-1):sum()}\r\n\tfor i = 1, xs:size(1) do\r\n\t\tys[i] = 0.5*((dists[{idxs[1],{}}]:lt(xs[i]):sum()-(dists:size(2)-total[1]))/total[1]+(dists[{idxs[2],{}}]:lt(xs[i]):sum()-(dists:size(2)-total[2]))/total[2])\r\n\tend\r\n\r\n\tlocal command = {xs,ys,'-'}\r\n\tgnuplot.raw('set title \"'..title..'\"')\r\n\tif not disp_key then \r\n\t\tgnuplot.raw('unset key')\r\n\telse\r\n\t\tgnuplot.raw('set key font \",6\" right bottom')\r\n\tend\r\n\tgnuplot.raw('set xrange [0:0.5]')\r\n\tgnuplot.raw('set yrange [0:1]')\r\n\tgnuplot.plot(unpack(command))\r\nend\r\n\r\nfunction utils.calculateMetrics(dists)\r\n\tgnuplot.raw('set bmargin 1')\r\n\tgnuplot.raw('set lmargin 3.2')\r\n\tgnuplot.raw('set rmargin 2')\r\n\tgnuplot.raw('set multiplot layout 2,3 title \"MPII Validation (PCKh)\"')\r\n\tgnuplot.raw('set xtics font \",6\"')\r\n\tgnuplot.raw('set xtics font \",6\"')\r\n\tdisplayPCKh(dists, {9,10}, 'Head')\r\n\tdisplayPCKh(dists, {2,5}, 'Knee')\r\n\tdisplayPCKh(dists, {1,6}, 'Ankle')\r\n\tgnuplot.raw('set tmargin 2.5')\r\n\tgnuplot.raw('set bmargin 1.5')\r\n\tdisplayPCKh(dists, {13,14}, 'Shoulder')\r\n\tdisplayPCKh(dists, {12,15}, 'Elbow')\r\n\tdisplayPCKh(dists, {11,16}, 'Wrist', true)\t\r\n\tgnuplot.raw('unset multiplot')\r\n\t\r\n    local threshold = 0.5\r\n    dists:apply(function(x)\r\n        if x>=0 and x<= threshold then \r\n            return 1\r\n        elseif x>threshold then \r\n            return 0\r\n        end\r\n    end)\r\n\r\n    local count = torch.zeros(16)\r\n    local sums = torch.zeros(16)\r\n    for i=1,16 do\r\n        dists[i]:apply(function(x)\r\n            if x ~= -1 then\r\n                count[i] = count[i] + 1\r\n                sums[i] = sums[i] + x\r\n            end\r\n        end)\r\n    end\r\n\r\n    local partNames = {'Head', 'Knee', 'Ankle', 'Shoulder', 'Elbow', 'Wrist', 'Hip'}\r\n    local partsC =  torch.Tensor({{9,10},{2,5},{1,6},{13,14},{12,15},{11,16},{3,4}})\r\n    print('PCKh results:')\r\n    for i=1,#partNames do\r\n        print(partNames[i]..': ',(sums[partsC[i][1]]/count[partsC[i][1]]+sums[partsC[i][2]]/count[partsC[i][1]])*100/2)\r\n    end\r\nend\r\n\r\nreturn utils\r\n"
  }
]