Awesome

pycutorch

POC for Python wrappers for cutorch/cunn

Example usage

Pre-requisites

• have cuda in /opt/cuda-7.0 (or modify setup.py to point to different path)
• Have installed torch, per https://github.com/torch/distro
• Have installed cutorch and cunn:

luarocks install cutorch
luarocks install cunn

• Have python 2.7
• Have setup a virtualenv, with cython, and numpy:

virtualenv env
source env/bin/activate
pip install cython
pip install numpy
pip install Jinja2

• You need to have pytorch repo next to pycutorch repository, ie the parent folder of pycutorch should contain pytorch

To run:

./build.sh
./run.sh

Example script:

from __future__ import print_function
import PyCudaTorch

#PyCudaTorch.newfunction(123)

import PyTorch
from PyTorchAug import *

def myeval(expr):
    print(expr, ':', eval(expr))

#a = PyTorch.foo(3,2)
#print('a', a)
#print(PyTorch.FloatTensor(3,2))

a = PyTorch.FloatTensor(4, 3).uniform()
print('a', a)
a = a.cuda()
print(type(a))

print('a.dims()', a.dims())
print('a.size()', a.size())
print('a', a)

print('sum:', a.sum())
myeval('a + 1')
b = PyCudaTorch.CudaTensor()
print('got b')
myeval('b')
b.resizeAs(a)
myeval('b')
print('run uniform')
b.uniform()
myeval('b')

print('create new b')
b = PyCudaTorch.CudaTensor()
print('b.dims()', b.dims())
print('b.size()', b.size())
print('b', b)

c = PyTorch.FloatTensor().cuda()
print('c.dims()', c.dims())
print('c.size()', c.size())
print('c', c)

print('creating Linear...')
linear = Linear(3,5)
print('created linear')
print('linear:', linear)
myeval('linear.output')
myeval('linear.output.dims()')
myeval('linear.output.size()')
myeval('linear.output.nElement()')

linear = linear.cuda()
myeval('type(linear)')
myeval('type(linear.output)')
myeval('linear.output.dims()')
myeval('linear.output.size()')
myeval('linear.output')
#print('linearCl.output', linear.output)

output = linear.forward(a)

print('output.dims()', output.dims())
print('output.size()', output.size())

outputFloat = output.float()
print('outputFloat', outputFloat)

print('output', output)

mlp = Sequential()
mlp.add(SpatialConvolutionMM(1,16,5,5,1,1,2,2))
mlp.add(ReLU())
mlp.add(SpatialMaxPooling(3,3,3,3))
mlp.add(SpatialConvolutionMM(16,32,5,5,1,1,2,2))
mlp.add(ReLU())
mlp.add(SpatialMaxPooling(2,2,2,2))
mlp.add(Reshape(32*4*4))
mlp.add(Linear(32*4*4, 150))
mlp.add(Tanh())
mlp.add(Linear(150, 10))
mlp.add(LogSoftMax())

mlp.cuda()

print('mlp', mlp)
myeval('mlp.output')
input = PyTorch.FloatTensor(128,1,28,28).uniform().cuda()
myeval('input[0][0][0]')
output = mlp.forward(input)
myeval('output[0]')


def test_pycudann():
#    PyTorch.manualSeed(123)
    linear = Linear(3, 5).cuda()
    print('linear', linear)
    print('linear.weight', linear.weight)
    print('linear.output', linear.output)
    print('linear.gradInput', linear.gradInput)

    input = PyTorch.DoubleTensor(4, 3).uniform().cuda()
    print('input', input)
    output = linear.updateOutput(input)
    print('output', output)

    gradInput = linear.updateGradInput(input, output)
    print('gradInput', gradInput)

    criterion = ClassNLLCriterion().cuda()
    print('criterion', criterion)

    print('dir(linear)', dir(linear))

    mlp = Sequential()
    mlp.add(linear)
    mlp.cuda()

    output = mlp.forward(input)
    print('output', output)

    import sys
    sys.path.append('../pytorch/thirdparty/python-mnist')
    from mnist import MNIST
    import numpy
    import array

#    numpy.random.seed(123)

    mlp = Sequential()

    mlp.add(SpatialConvolutionMM(1, 16, 5, 5, 1, 1, 2, 2))
    mlp.add(ReLU())
    mlp.add(SpatialMaxPooling(3, 3, 3, 3))

    mlp.add(SpatialConvolutionMM(16, 32, 3, 3, 1, 1, 1, 1))
    mlp.add(ReLU())
    mlp.add(SpatialMaxPooling(2, 2, 2, 2))

    mlp.add(Reshape(32 * 4 * 4))
    mlp.add(Linear(32 * 4 * 4, 150))
    mlp.add(Tanh())
    mlp.add(Linear(150, 10))
    mlp.add(LogSoftMax())
    mlp.cuda()

    criterion = ClassNLLCriterion().cuda()
    print('got criterion')

    learningRate = 0.02

    mndata = MNIST('/norep/data/mnist')
    imagesList, labelsB = mndata.load_training()
    images = numpy.array(imagesList).astype(numpy.float64)
    #print('imagesArray', images.shape)

    #print(images[0].shape)

    labelsf = array.array('d', labelsB.tolist())
    imagesTensor = PyTorch.asDoubleTensor(images).cuda()

    #imagesTensor = PyTorch.FloatTensor(100,784)
    #labels = numpy.array(20,).astype(numpy.int32)
    #labelsTensor = PyTorch.FloatTensor(100).fill(1)
    #print('labels', labels)
    #print(imagesTensor.size())

    def printStorageAddr(name, tensor):
        print('printStorageAddr START')
        storage = tensor.storage()
        if storage is None:
            print(name, 'storage is None')
        else:
            print(name, 'storage is ', hex(storage.dataAddr()))
        print('printStorageAddr END')

    labelsTensor = PyTorch.asDoubleTensor(labelsf).cuda()
    labelsTensor += 1
    #print('calling size on imagestensor...')
    #print('   (called size)')

    desiredN = 1280
    maxN = int(imagesTensor.size()[0])
    desiredN = min(maxN, desiredN)
    imagesTensor = imagesTensor.narrow(0, 0, desiredN)
    labelsTensor = labelsTensor.narrow(0, 0, desiredN)
    print('imagesTensor.size()', imagesTensor.size())
    print('labelsTensor.size()', labelsTensor.size())
    N = int(imagesTensor.size()[0])
    print('type(imagesTensor)', type(imagesTensor))
    size = PyTorch.LongStorage(4)
    size[0] = N
    size[1] = 1
    size[2] = 28
    size[3] = 28
    imagesTensor.resize(size)
    imagesTensor /= 255.0
    imagesTensor -= 0.2
    print('imagesTensor.size()', imagesTensor.size())

    print('start training...')
    for epoch in range(12):
        numRight = 0
        for n in range(N):
    #        print('n', n)
            input = imagesTensor[n]
            label = labelsTensor[n]
            labelTensor = PyTorch.DoubleTensor(1).cuda()
            labelTensor[0] = label
    #        print('label', label)
            output = mlp.forward(input)
            prediction = PyCudaTorch.getPrediction(output)
    #        print('prediction', prediction)
            if prediction == label:
                numRight += 1
            criterion.forward(output, labelTensor)
            mlp.zeroGradParameters()
            gradOutput = criterion.backward(output, labelTensor)
            mlp.backward(input, gradOutput)
            mlp.updateParameters(learningRate)
    #        PyTorch.collectgarbage()
    #        if n % 100 == 0:
    #            print('n=', n)
        print('epoch ' + str(epoch) + ' accuracy: ' + str(numRight * 100.0 / N) + '%')

if __name__ == '__main__':
    test_pycudann()

Output:

$ ./run.sh 
/home/user/envs/local/lib/python2.7/site-packages/pkg_resources.py:991: UserWarning: /home/user/.python-eggs is writable by group/others and vulnerable to attack when used with get_resource_filename. Consider a more secure location (set with .set_extraction_path or the PYTHON_EGG_CACHE environment variable).
  warnings.warn(msg, UserWarning)
dir(PyTorchAug) ['ClassNLLCriterion', 'Linear', 'LogSoftMax', 'LuaClass', 'MSECriterion', 'PyTorch', 'ReLU', 'Reshape', 'Sequential', 'SpatialConvolutionMM', 'SpatialMaxPooling', 'Tanh', '__builtins__', '__doc__', '__file__', '__loader__', '__name__', '__package__', 'cythonClasses', 'getNextObjectId', 'lua', 'luaClasses', 'luaClassesReverse', 'nextObjectId', 'popString', 'populateLuaClassesReverse', 'print_function', 'pushFunctionByPythonClass', 'pushGlobal', 'pushGlobalFromList', 'pushObject', 'registerObject', 'torchType', 'unregisterObject']
dir(PyCudaTorch) ['CudaGlobalState', 'CudaTensor', 'DoubleTensorToCudaTensor', 'FloatTensorToCudaTensor', 'PyTorch', '__builtins__', '__doc__', '__file__', '__name__', '__package__', 'array', 'cyPopCudaTensor', 'cyPushCudaTensor', 'getPrediction', 'numbers']
initializing PyCudaTorch...
loaded cutorch
loaded cunn
 ... PyCudaTorch initialized
a 0.682399 0.991263 0.193337
0.771042 0.859513 0.227952
0.0226214 0.722964 0.338015
0.353466 0.758877 0.573851
[torch.FloatTensor of size 4x3]

<type 'PyCudaTorch.CudaTensor'>
a.dims() 2
a.size()  4
 3
[torch.LongStorage of size 2]

a 0.682399 0.991263 0.193337
0.771042 0.859513 0.227952
0.0226214 0.722964 0.338015
0.353466 0.758877 0.573851
[torch.CudaTensor of size 4x3]

sum: 6.49530076981
a + 1 : 1.6824 1.99126 1.19334
1.77104 1.85951 1.22795
1.02262 1.72296 1.33801
1.35347 1.75888 1.57385
[torch.CudaTensor of size 4x3]

got b
b : [torch.CudaTensor with no dimension]

b : 1.6824 1.99126 1.19334
1.77104 1.85951 1.22795
1.02262 1.72296 1.33801
1.35347 1.75888 1.57385
[torch.CudaTensor of size 4x3]

run uniform
b : 0.245807 0.927532 0.151811
0.720315 0.144195 0.360412
0.122866 0.823565 0.643401
0.374496 0.192979 0.93561
[torch.CudaTensor of size 4x3]

create new b
b.dims() 0
b.size() [torch.LongStorage of size 0]

b [torch.CudaTensor with no dimension]

c.dims() 0
c.size() [torch.LongStorage of size 0]

c [torch.CudaTensor with no dimension]

creating Linear...
Linear.__init__
created linear
linear: nn.Linear(3 -> 5)
linear.output : [torch.DoubleTensor with no dimension]

linear.output.dims() : 0
linear.output.size() : None
linear.output.nElement() : 0
Linear.__init__
type(linear) : <class 'PyTorchAug.Linear'>
type(linear.output) : <type 'PyCudaTorch.CudaTensor'>
linear.output.dims() : 0
linear.output.size() : [torch.LongStorage of size 0]

linear.output : [torch.CudaTensor with no dimension]

output.dims() 2
output.size()  4
 5
[torch.LongStorage of size 2]

outputFloat 0.835138 -0.0984995 0.976471 0.00963038 0.0140738
0.841528 -0.0715615 0.96343 0.0199725 -0.018722
0.665126 -0.0909479 0.698118 -0.12425 0.199515
0.867903 -0.20736 0.928468 -0.0403201 -0.0103005
[torch.FloatTensor of size 4x5]

output 0.835138 -0.0984995 0.976471 0.00963038 0.0140738
0.841528 -0.0715615 0.96343 0.0199725 -0.018722
0.665126 -0.0909479 0.698118 -0.12425 0.199515
0.867903 -0.20736 0.928468 -0.0403201 -0.0103005
[torch.CudaTensor of size 4x5]

Linear.__init__
Linear.__init__
mlp nn.Sequential {
  [input -> (1) -> (2) -> (3) -> (4) -> (5) -> (6) -> (7) -> (8) -> (9) -> (10) -> (11) -> output]
  (1): nn.SpatialConvolutionMM(1 -> 16, 5x5, 1,1, 2,2)
  (2): nn.ReLU
  (3): nn.SpatialMaxPooling(3,3,3,3)
  (4): nn.SpatialConvolutionMM(16 -> 32, 5x5, 1,1, 2,2)
  (5): nn.ReLU
  (6): nn.SpatialMaxPooling(2,2,2,2)
  (7): nn.Reshape(512)
  (8): nn.Linear(512 -> 150)
  (9): nn.Tanh
  (10): nn.Linear(150 -> 10)
  (11): nn.LogSoftMax
}
mlp.output : [torch.CudaTensor with no dimension]

input[0][0][0] : 0.951148 0.383791 0.927399 0.896064 0.918963 0.757004 0.527258 0.808004 0.533048 0.880753 0.392538 0.858083 0.204616 0.0887245 0.881087 0.6978 0.717542 0.333986 0.723461 0.970431 0.806659 0.431718 0.32057 0.0817998 0.766709 0.390999 0.0842737 0.936283
[torch.CudaTensor of size 28]

output[0] : -2.41201 -2.48542 -2.16704 -2.25163 -2.26283 -2.21121 -2.34368 -2.32936 -2.41525 -2.19796
[torch.CudaTensor of size 10]

Linear.__init__
Linear.__init__
linear nn.Linear(3 -> 5)
linear.weight 0.126199 0.255241 -0.0918763
-0.34899 0.532427 0.0729166
-0.432081 0.354608 -0.539946
-0.405352 -0.00362546 0.51926
-0.10635 -0.538465 -0.411125
[torch.CudaTensor of size 5x3]

linear.output [torch.CudaTensor with no dimension]

linear.gradInput [torch.CudaTensor with no dimension]

input 0.527763 0.92302 0.252646
0.260251 0.346376 0.156644
0.562014 0.704626 0.220798
0.272335 0.334772 0.0299162
[torch.CudaTensor of size 4x3]

output -0.184994 0.00507969 -0.591282 -0.138991 -0.610176
-0.357116 -0.215582 -0.628341 -0.0783141 -0.231755
-0.233488 -0.125474 -0.666329 -0.16862 -0.483128
-0.34691 -0.235218 -0.569251 -0.148975 -0.17469
[torch.CudaTensor of size 4x5]

gradInput 0.351596 0.0748752 0.515313
0.358055 -0.303671 0.410976
0.421963 -0.101929 0.483152
0.363238 -0.321038 0.316549
[torch.CudaTensor of size 4x3]

criterion nn.ClassNLLCriterion
dir(linear) ['addBuffer', 'bias', 'gradBias', 'gradInput', 'gradWeight', 'output', 'weight']
output -0.184994 0.00507969 -0.591282 -0.138991 -0.610176
-0.357116 -0.215582 -0.628341 -0.0783141 -0.231755
-0.233488 -0.125474 -0.666329 -0.16862 -0.483128
-0.34691 -0.235218 -0.569251 -0.148975 -0.17469
[torch.CudaTensor of size 4x5]

Linear.__init__
Linear.__init__
got criterion
imagesTensor.size()  1280
 784
[torch.LongStorage of size 2]

labelsTensor.size()  1280
[torch.LongStorage of size 1]

type(imagesTensor) <type 'PyCudaTorch.CudaTensor'>
imagesTensor.size()  1280
 1
 28
 28
[torch.LongStorage of size 4]

start training...
epoch 0 accuracy: 64.453125%
epoch 1 accuracy: 91.953125%
epoch 2 accuracy: 96.015625%
epoch 3 accuracy: 97.890625%
epoch 4 accuracy: 99.453125%
epoch 5 accuracy: 99.765625%
epoch 6 accuracy: 99.84375%
epoch 7 accuracy: 99.84375%
epoch 8 accuracy: 99.84375%
epoch 9 accuracy: 99.84375%
epoch 10 accuracy: 99.921875%
epoch 11 accuracy: 100.0%