Tutorial about efficiently passing python list or array
sk-Prime opened this issue · 5 comments
first of all, it is an awesome package. I have experience in python and new to nim. nimpy is the easiest way to combine the power of two languages.
but i am looking for some documents regarding passing various data type from python to nim and returning them. My question is what is the efficient way to pass let say python list without making copy of it? or python Pillow pixel data (efficient way to pass image pixel to nimpy will be fantastic). some tutorial will be helpful about it.
by the way thanks for the hard-work for nimpy.
Well the answer is in the readme. Let me cite it:
nimpy allows manipulating numpy objects just how you would do it in Python, however it is not much more efficient. To get the maximum performance nimpy exposes Buffer protocol, see raw_buffers.nim. tpyfromnim.nim contains a very basic test for this (grep numpy). Higher level API might be considered in the future, PRs are welcome.
I saw your message on IRC earlier (at least I assume that was you; owl_000).
I wanted to see if I could get something like this to work. Came up with the following. But probably don't trust me on this, I might be doing something really stupid in there. :)
Just did a quick test and seems to be about as fast as doing the same with numpy directly (dd = dd // 2) (which was actually surprising to me).
I actually thought that arraymancer had some form of fromBuffer proc to create a tensor from a raw buffer for better handling, but couldn't find it.
import nimpy
import nimpy / [raw_buffers, py_types]
import seqmath, sequtils
type
NumpyArray[T] = object
pyBuf: ptr RawPyBuffer # to keep track of the buffer so that we can release it
data: ptr UncheckedArray[T] # this will be the raw data
shape: seq[int]
len: int
proc initNdArray[T](ar: PyObject): NumpyArray[T] =
## some PyObject that points to a numpy array
## User has to make sure that the data type of the array can be
## cast to `T` without loss of information!
result.pyBuf = cast[ptr RawPyBuffer](alloc0(sizeof(RawPyBuffer)))
ar.getBuffer(result.pyBuf[], PyBUF_WRITABLE or PyBUF_ND)
let shapear = cast[ptr UncheckedArray[Py_ssize_t]](result.pyBuf.shape)
for i in 0 ..< result.pyBuf.ndim:
let dimsize = shapear[i].int # py_ssize_t is csize
result.shape.add dimsize
result.len = result.shape.foldl(a * b, 1)
result.data = cast[ptr UncheckedArray[T]](result.pyBuf.buf)
proc release[T](nd: var NumpyArray[T]) =
## releases the data buffer
nd.pyBuf[].release()
dealloc(nd.pyBuf)
template parseIndices(indices: varargs[int], shape: seq[int]): int =
## parses the indices and returns a single integer corresponding to memory loc
var res: int
var stride = 1
for axis, idx in indices:
if axis == 0:
res = idx
stride = shape[axis]
else:
res += (stride * idx)
stride *= shape[axis]
res
proc `[]`[T](nd: NumpyArray[T], indices: varargs[int]): T =
## index points to single element. One index per dimension of array
assert indices.len == nd.shape.len
let index = parseIndices(indices, nd.shape)
if index > nd.len or index < 0:
raise newException(IndexError, "Bad index " & $index & " for array of shape " &
$nd.shape)
result = nd.data[index]
proc `[]=`[T](nd: var NumpyArray[T], indices: varargs[int], val: T) =
## index points to single element. One index per dimension of array
assert indices.len == nd.shape.len
let index = parseIndices(indices, nd.shape)
if index > nd.len or index < 0:
raise newException(IndexError, "Bad index " & $index & " for array of shape " &
$nd.shape)
nd.data[index] = val
proc takeBuffer(a: PyObject) =
var dd = initNdArray[uint8](a)
for z in 0 ..< dd.shape[0]:
for y in 0 ..< dd.shape[1]:
for x in 0 ..< dd.shape[2]:
dd[z, y, x] = dd[z, y, x] div 2
dd.release()
let Image = pyImport("PIL.Image")
let np = pyImport("numpy")
# open some image
let im = Image.open("axion_conversion_prob_0.tiff")
discard im.show()
# make a numpy array from it
let imnp = np.array(im)
# and do some stuff with it using raw buffers
takeBuffer(imnp)
# create a new image from the modified buffer and save and show
let newImg = Image.fromarray(imnp)
discard newImg.show()
discard newImg.save("newfile.tiff")@Vindaar please make a git package to handle Python PiL library using nimpy. that will be great for beginner like me.
Since I just implemented fromBuffer in the arraymancer PR that makes tensors ptr + len pairs:
mratsim/Arraymancer#477
I thought it would be fun to revisit the code snippet from above to see what we can do and to also make use of ARC so we don't have to release the RawPyBuffer manually:
import nimpy
import nimpy / [raw_buffers, py_types]
import seqmath, sequtils, times
import arraymancer
type
SafePyBuffer = ref SafePyBufferObj
SafePyBufferObj = object
pyBuf: RawPyBuffer
shape: seq[int]
when defined(gcDestructors):
proc `=destroy`(s: var SafePyBufferObj) =
if not s.pyBuf.buf.isNil:
s.pyBuf.release()
else:
proc finalizer(s: SafePyBuffer) =
if not s.isNil and not s.pyBuf.buf.isNil:
s.pyBuf.release()
template benchmark(body: untyped): untyped =
let t0 = cpuTime()
for _ in 0 ..< 1000:
body
let t1 = cpuTime()
echo "Took ", t1 - t0, " and per iteration ", ((t1 - t0) / 1000.0) * 1e6, " µs"
proc newSafePyBuffer(a: PyObject): SafePyBuffer =
when defined(gcDestructors):
new(result)
else:
new(result, finalizer)
var buf: RawPyBuffer
getBuffer(a, buf, PyBUF_WRITABLE or PyBUF_ND)
let shapeArray = cast[ptr UncheckedArray[Py_ssize_t]](buf.shape)
result.shape = newSeq[int](buf.ndim)
for i in 0 ..< buf.ndim:
result.shape[i] = shapeArray[i].int # py_ssize_t is csize
result.pyBuf = buf
proc toTensor[T](s: SafePyBuffer): Tensor[T] =
result = fromBuffer[T](s.pyBuf.buf, s.shape)
proc modifyBuffer[T](t: var Tensor[T]) =
benchmark: # take out the benckmark to get something useful ;)
t.apply_inline(x div 2)
let Image = pyImport("PIL.Image")
let np = pyImport("numpy")
# open some image
let im = Image.open("axion_conversion_prob_0.tiff")
discard im.show()
# make a numpy array from it
let imnp = np.array(im)
# and do some stuff with it using raw buffers
let sb = newSafePyBuffer(imnp)
var t = toTensor[uint8](sb)
modifyBuffer(t)
# thanks to finalizer / destructor no releasing of the PyBuffer required.
# but: if your SafePyBuffer were to leave the scope and thus release
# the buffer before the `Tensor` we created, the tensors data would
# be invalid!
# create a new image from the modified buffer and save and show
let newImg = Image.fromarray(imnp)
discard newImg.show()
discard newImg.save("newfile.tiff")Turning a numpy array into something "safe" and accessing its shape is still the most "complex" part (newSafePyBuffer). The toTensor proc now is just a single line, taking the shape and raw pointer to the data. Zero copy required and every arraymancer feature available!
Seems to work perfectly fine. SafePyBuffer is something that might be worthwhile to be implemented into nimpy imo.
Of course this is pending the PR mentioned at the top, but that should be merged soon.
From here the user is in principle only supposed to write the code from below toTensor. So usability and performance are excellent and having immediate feature parity with a "native Nim Tensor" is great.
Keep in mind though that the separation between the Python buffers and the Tensor means there's some inherent "unsafety". Namely if the SafePyBuffer runs out of scope and is destroyed, but the tensor lives on! Therefore there's maybe some bikeshedding to be done about possibly combining the tensor and safe buffer into one type (which would require the user to manually access the tensor field of the safe buffer when wanting to use it, which is not ideal).
Relevant to this topic, but I wrote some code to easily interact with Numpy Array and Arraymancer / ptr UncheckedArray[T] .
You can find the code here :https://github.com/SciNim/scinim/blob/main/scinim/numpyarrays.nim . It's mostly based on @Vindaar 's sample above, slightly improved. It just feels better to centralize it somewhere so we don't re-invent the wheel each time