Python in High-Performance Computing

Jussi Enkovaara
Martti Louhivuori
Outline
NumPy fast array interface for Python
MP!"Py MP! interface for Python
E#ten$ing Python %ith C
Numpy
Numpy fast array interface
&tan$ar$ Python is not %ell suita'le for
numerical computations
lists are very fle#i'le 'ut also slo% to process
in numerical computations
Numpy a$$s a ne% array $ata type
static( multi$imensional
fast processing of arrays
some linear alge'ra( ran$om num'ers
Numpy arrays
)ll elements of an array have the same type
)rray can have multiple $imensions
*he num'er of elements in the array is
fi#e$( shape can 'e change$
Creating numpy arrays
+rom a list
,,, import numpy as np
,,, a - np.array//0( 1( 2( "3( float3
,,, a
array/4 0.( 1.( 2.( ".53
,,, list0 - 440( 1( 25( 4"(6(755
,,, mat - np.array/list0( comple#3
,,, mat
array/44 0.89.:( 1.89.:( 2.89.:5(
4 ".89.:( 6.89.:( 7.89.:553
,,, mat.shape
/1( 23
,,, mat.si;e
7
Creating numpy arrays
More %ays for creating arrays
,,, import numpy as np
,,, a - np.arange/093
,,, a
array/49( 0( 1( 2( "( 6( 7( <( =( >53
,,, ' - np.linspace/-".6( ".6( 63
,,, '
array/4-".6 ( -1.16( 9. ( 1.16( ".6 53
,,, c - np.;eros//"( 73( float3
,,, c.shape
/"( 73
,,, $ - np.ones//1( "33
,,, $
array/44 0.( 0.( 0.( 0.5(
4 0.( 0.( 0.( 0.553
!n$e#ing an$ slicing arrays
&licing is possi'le over all $imensions
,,, a - np.arange/093
,,, a40?<?15
array/40( 2( 653
,,, a - np.;eros//"( "33
,,, a40?2( 0?25 - 1.9
,,, a
array/44 9.( 9.( 9.( 9.5(
4 9.( 1.( 1.( 9.5(
4 9.( 1.( 1.( 9.5(
4 9.( 9.( 9.( 9.553
&imple in$e#ing
,,, mat - np.array/440( 1( 25( 4"( 6( 7553
,,, mat49(15
2
,,, mat40(-15
,,, 6
@ie%s an$ copies of arrays
&imple assignment creates references to
arrays
&licing creates Avie%sB to the arrays
Cse copy/3 for real copying of arrays
a = np.arange(10)
b = a # reference, changing values in b changes a
b = a.copy() # true copy
c = a[1:4] # view, changing c changes elements [1:4] of a
c = a[1:4].copy() # true copy of subarray
)rray manipulation
reshape ? change the shape of array
,,, mat - np.array/440( 1( 25( 4"( 6( 7553
,,, mat
array/440( 1( 25(
4"( 6( 7553
,,, mat.reshape/2(13
array/440( 15(
42( "5(
46( 7553
ravel ? flatten array to 0-$
reshape ? change the shape of array
,,, mat.ravel/3
array/40( 1( 2( "( 6( 753
)rray manipulation
concatenate ? :oin arrays together
,,, mat0 - np.array/440( 1( 25( 4"( 6( 7553
,,, mat1 - np.array/44<( =( >5( 409( 00( 01553
,,, np.concatenate//mat0( mat133
array/44 0( 1( 25(
4 "( 6( 75(
4 <( =( >5(
409( 00( 01553
,,, np.concatenate//mat0( mat13( a#is-03
array/44 0( 1( 2( <( =( >5(
4 "( 6( 7( 09( 00( 01553
split ? split array to N pieces
,,, np.split/mat0( 2( a#is-03
4array/4405(
4"553( array/4415(
46553( array/4425(
475535
)rray operations
Most operations for numpy arrays are $one
element-%ise
8( -( D( E( DD
>>> a = np.array([1.0, 2.0, 3.0])
>>> b = 2.0
>>> a * b
array([ 2., 4., 6.])
>>> a + b
array([ 3., 4., 5.])
>>> a * a
array([ 1., 4., 9.])
)rray operations
Numpy has special functions %hich can
%ork %ith array arguments
sin( cos( e#p( sFrt( log( ...
,,, import numpy( math
,,, a - numpy.linspace/-pi( pi( =3
,,, a
array/4-2.0"06>176( -1.1"2>>"<6( -0.2"72>7=6( -9.""=<>=>6(
9.""=<>=>6( 0.2"72>7=6( 1.1"2>>"<6( 2.0"06>17653
,,, math.sin/a3
*race'ack /most recent call last3?
+ile GHst$in,G( line 0( in I
*ypeError? only length-0 arrays can 'e converte$ to Python scalars
,,, numpy.sin/a3
array/4 -0.11"7"7=9e-07( -<.=0=20"=1e-90( ->.<">1<>01e-90(
-".22==2<2>e-90( ".22==2<2>e-90( >.<">1<>01e-90(
<.=0=20"=1e-90( 0.11"7"7=9e-0753
@ectori;e$ operations
for loops in Python are slo%
Cse Avectori;e$B operations %hen possi'le
E#ample? $ifference
vs.
for loop is J=9 times slo%erK
arr = np.arange(1000)
dif = np.zeros(999, int)
for i in range(1, len(arr)):
dif[i-1] = arr[i] - arr[i-1]
arr = np.arange(1000)
dif = arr[1:] - arr[:-1]
Lroa$casting
!f array shapes are $ifferent( the smaller
array may 'e 'roa$caste$ into a larger
shape
>>> from numpy import array
>>> a = array([[1,2],[3,4],[5,6]], float)
>>> a
array([[ 1., 2.],
[ 3., 4.],
[ 5., 6.]])
>>> b = array([[7,11]], float)
>>> b
array([[ 7., 11.]])
>>> a * b
array([[ 7., 22.],
[ 21., 44.],
[ 35., 66.]])
)$vance$ in$e#ing
Numpy arrays can 'e in$e#e$ also %ith
other arrays /integer or 'oolean3
>>> x = np.arange(10,1,-1)
>>> x
array([10, 9, 8, 7, 6, 5, 4, 3, 2])
>>> x[np.array([3, 3, 1, 8])]
array([7, 7, 9, 2])
Loolean AmaskB arrays
>>> m = x > 7
>>> m
array([ True, True, True, False, False, ...
>>> x[m]
array([10, 9, 8])
)$vance$ in$e#ing creates copies of arrays
Maske$ arrays
&ometimes $atasets contain invali$ $ata
/faulty measurement( pro'lem in simulation3
Maske$ arrays provi$e a %ay to perform
array operations neglecting invali$ $ata
Maske$ array support is provi$e$ 'y
numpy.ma mo$ule
Maske$ arrays
Maske$ arrays can 'e create$ 'y
com'ining a regular numpy array an$ a
'oolean mask
>>> import numpy.ma as ma
>>> x = np.array([1, 2, 3, -1, 5])
>>> m = x < 0
>>> mx = ma.masked_array(x, mask=m)
>>> mx
masked_array(data = [1 2 3 -- 5],
mask = [False False False True False],
fill_value = 999999)
>>> x.mean()
2.0
>>> mx.mean()
2.75
!EO %ith Numpy
Numpy provi$es functions for rea$ing $ata
from file an$ for %riting $ata into the files
&imple te#t files
numpy.loa$t#t
numpy.savet#t
Mata in regular column layout
Can $eal %ith comments an$ $ifferent column
$elimiters
Nan$om num'ers
*he mo$ule numpy.ran$om provi$es
several functions for constructing ran$om
arrays
ran$om? uniform ran$om num'ers
normal? normal $istri'ution
poisson? Poisson $istri'ution
...
,,, import numpy.ran$om as rn$
,,, rn$.ran$om//1(133
array/44 9.91>9>0"1( 9.>9="= 5(
4 9.>"<020" ( 9.20"1"2>2553
,,, rn$.poisson/si;e-/1(133
Polynomials
Polynomial is $efine$ 'y array of coefficients p
p/#( N3 - p495 #
N-0
8 p405 #
N-1
8 ... 8 p4N-05
Least sFuare fitting? numpy.polyfit
Evaluating polynomials? numpy.polyval
Noots of polynomial? numpy.roots
...
,,, # - np.linspace/-"( "( <3
,,, y - #DD1 8 rn$.ran$om/#.shape3
,,,
,,, p - np.polyfit/#( y( 13
,,, p
array/4 9.>7=7>992( -9.9006<1<6( 9.7>26160"53
Linear alge'ra
Numpy can calculate matri# an$ vector pro$ucts
efficiently
$ot( v$ot( ...
Eigenpro'lems
linalg.eig( linalg.eigvals( O
Linear systems an$ matri# inversion
linalg.solve( linalg.inv
,,, ) - np.array///1( 03( /0( 2333
,,, L - np.array///-1( ".13( /".1( 7333
,,, C - np.$ot/)( L3
,,, ' - np.array//0( 133
,,, np.linalg.solve/C( '3 P solve C # - '
array/4 9.9""62""0( 9.97==16>053
Numpy performance
Matri# multiplication
C - ) D L
matri# $imension 199
pure python? 6.29 s
naive C? 9.9> s
numpy.$ot? 9.90 s
&ummary
Numpy provi$es a static array $ata
structure
Multi$imensional arrays
+ast mathematical operations for arrays
)rrays can 'e 'roa$caste$ into same
shapes
*ools for linear alge'ra an$ ran$om
num'ers
Parallel programming %ith
Python using mpi"py
Outline
Lrief intro$uction to message passing
interface /MP!3
Python interface to MP! mpi"py
Performance consi$erations
Message passing interface
MP! is an application programming interface /)P!3 for communication
'et%een separate processes
*he most %i$ely use$ approach for $istri'ute$ parallel computing
MP! programs are porta'le an$ scala'le
the same program can run on $ifferent types of computers( from
PCQs to supercomputers
MP! is fle#i'le an$ comprehensive
large /over 019 proce$ures3
concise /often only 7 proce$ures are nee$e$3
MP! stan$ar$ $efines C an$ +ortran interfaces
mpi4py provi$es /an unofficial3 Python interface
E#ecution mo$el in MP!
Parallel program is launche$ as set of independent, identical
processes
Parallel program
Process 0
Process 1
Process N
...
)ll the processes contain the same program co$e an$ instructions
Processes can resi$e in $ifferent no$es or even in $ifferent computers
*he %ay to launch parallel program is implementation $epen$ent
mpirun( mpie#ec( aprun( poe( ...
Rhen using Python( one launches N Python interpreters
mpirun -np 21 python parallelSscript.py
MP! Concepts
rank? i$ num'er given to process
it is possi'le to Fuery for rank
processes can perform $ifferent tasks
'ase$ on their rank
Communicator? group containing process
in mpi"py the 'asic o':ect %hose
metho$s are calle$
MPI_COMM_WORLD contains all the
process /MP!.COMMSRONLM in mpi"py3
...
if ( rank == 0 )


if ( rank == 1)


...
Mata mo$el
)ll varia'les an$ $ata structures are local to the process
Processes can e#change $ata 'y sen$ing an$ receiving messages
a - 0.9
' - 1.9
a - -0.9
' - -1.9
Messages
MPI
Process 0
/rank 9 3
Process 1
/rank 0 3
...
a - 7.9
' - 6.9
Process N
/rank N-0 3
Csing mpi"py
Lasic metho$s of communicator o':ect
TetSsi;e/3
Num'er of processes in communicator
TetSrank/3
rank of this process
from mpi4py import MPI
comm = MPI.COMM_WORLD # Communicator object containing all
# processes
from mpi4py import MPI
comm = MPI.COMM_WORLD
size = comm.Get_size()
rank = comm.Get_rank()
print I am rank %d in group of %d processes % (rank, size)
&en$ing an$ receiving $ata
&en$ing an$ receiving a $ictionary
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
if rank == 0:
data = {'a': 7, 'b': 3.14}
comm.send(data, dest=1, tag=11)
elif rank == 1:
data = comm.recv(source=0, tag=11)
&en$ing an$ receiving $ata
)r'itrary Python o':ects can 'e communicate$ %ith
the send an$ receive metho$s of communicator
send(data, dest, tag)
data Python o':ect to sen$
dest $estination rank
tag i$ given to the message
receive(sorce, tag)
sorce source rank
tag i$ given to the message
$ata is provi$e$ as return value
Mestination an$ source ranks as %ell as tags have to
match
Communicating NumPy
arrays
)r'itrary Python o':ects are converte$ to 'yte
streams %hen sen$ing
Lyte stream is converte$ 'ack to Python o':ect %hen
receiving
Conversions give overhea$ to communication
/Contiguous3 NumPy arrays can 'e communicate$
%ith very little overhea$ %ith upper case metho$s?
!en$/$ata( $est( tag3
Recv/$ata( source( tag3
Note the $ifference in receiving? the $ata array has
to e#ist in the time of call
Communicating NumPy
arrays
from mpi4py import MPI
import numpy
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
if rank == 0:
data = numpy.arange(100, dtype=numpy.float)
comm.Send(data, dest=1, tag=13)
elif rank == 1:
data = numpy.empty(100, dtype=numpy.float)
comm.Recv(data, source=0, tag=13)
Note the $ifference 'et%een upperElo%er caseK
sen$Erecv? general Python o':ects( slo%
&en$ENecv? continuous arrays( fast
mpi"py performance
Ping-pong test
&ummary
mpi"py provi$es Python interface to MP!
MP! calls via communicator o':ect
Possi'le to communicate ar'itrary Python
o':ects
NumPy arrays can 'e communicate$ %ith
nearly same spee$ as from CE+ortran
C - e#tensions
C - e#tensions
&ome times there are time critical parts of co$e
%hich %oul$ 'enefit from compile$ language
>9E09 rule? >9 U of time is spent in
09 U of co$e
only a small part of application 'enefits from
compile$ co$e
!t is relatively straightfor%ar$ to create a Python
interface to C-functions
$ata is passe$ from Python( routine is e#ecute$
%ithout any Python overhea$s
C - e#tensions
C routines are 'uil$ into a share$ li'rary
Noutines are loa$e$ $ynamically %ith
normal import statements
) li'rary "ello#so is looke$ for
) function "ello /$efine$ in mye#t.so3 is
calle$
import hello
hello.hello()
Creating C-e#tension
03 !nclu$e Python hea$ers
13 Mefine the C-function
*ype of function is al%ays PyO':ect
+unction arguments are al%ays the same
/args is use$ for passing $ata from Python to C3
) macro pySNE*CNNSNONE is use$ for returning Anothing
#include <Python.h>
...
PyObject* hello_c(PyObject *self, PyObject *args)
{
printf("Hello\n");
Py_RETURN_NONE;
}
Creating C-e#tension
23 Mefine the Python interfaces for functions
"ello is the function name use$ in Python co$e(
"ello_c is the actual C-function to 'e calle$
&ingle e#tension mo$ule can contain several functions
/hello( func1( ...3
...
static PyMethodDef functions[] = {
{"hello", hello_c, METH_VARARGS, 0},
{func2, func2, METH_VARARGS, 0},
{0, 0, 0, 0} /* Sentinel notifies the end of definitions */
};
Creating C-e#tension
"3 Mefine the mo$ule initiali;ation function
E#tension mo$ule shoul$ 'e 'uil$ into "ello#so
E#tension is mo$ule is importe$ as import "ello
+unctionsEinterfaces $efine$ in functions are calle$ as
hello.hello/3( hello.func1/3( ...
63 Compile as share$ li'rary
*he location of Python hea$ers /EusrEinclu$eE...3 may
vary in $ifferent systems
...
PyMODINIT_FUNC inithello(void)
{
(void) Py_InitModule("hello", functions);
}
gcc -share$ -o mye#t.so -!EusrEinclu$eEpython1.7 -fP!C mye#t.c
+ull listing of hello.c
#include <Python.h>
PyObject* hello_c(PyObject *self, PyObject *args)
{
printf("Hello\n");
Py_RETURN_NONE;
}
static PyMethodDef functions[] = {
{"hello", hello_c, METH_VARARGS, 0},
{0, 0, 0, 0}
};
PyMODINIT_FUNC inithello(void)
{
(void) Py_InitModule("hello", functions);
}
Passing arguments to C-
functions
...
PyObject* my_C_func(PyObject *self, PyObject *args)
{
int a;
double b;
char* str;
if (!PyArg_ParseTuple(args, "ids", &a, &b, &str))
return NULL;
printf(int %i, double %f, string %s\n, a, b, str);
Py_RETURN_NONE;
}
Py$rg_Parse%ple checks that function is calle$ %ith
proper arguments
Ai$sB ? integer( $ou'le( string
an$ $oes the conversion from Python to C types
Neturning values
...
PyObject* square(PyObject *self, PyObject *args)
{
int a;
if (!PyArg_ParseTuple(args, "i", &a))
return NULL;
a = a*a;
return Py_BuildValue("i", a);
}
Create an$ return Python integer from C varia'le a
A$B %oul$ create Python $ou'le etc.
Neturning tuple?
Py_&ild'ale(((ids)(, a, ), str)*
Operating %ith NumPy array
#include <Python.h>
#define NO_IMPORT_ARRAY
#include <numpy/arrayobject.h>
PyObject* my_C_func(PyObject *self, PyObject *args)
{
PyArrayObject* a;
if (!PyArg_ParseTuple(args, "O", &a))
return NULL;
int size = PyArray_SIZE(a); /* Total size of array */
double *data = PyArray_DATA(a); /* Pointer to data */
for (int i=0; i < size; i++)
{
data[i] = data[i] * data[i];
}
Py_RETURN_NONE;
}
NumPy provi$es )P! also for $etermining the $imensions
of array etc.
*ools for easier interfacing
Cython
&R!T
pyre#
f1py /for +ortran co$e3
&ummary
Python can 'e e#ten$e$ %ith C-functions
relatively easily
C-e#tension 'uil$ as share$ li'rary
!t is possi'le to pass $ata 'et%een Python
an$ C co$e
E#ten$ing Python?
http?EE$ocs.python.orgEe#ten$ingE
NumPy C-)P!
http?EE$ocs.scipy.orgE$ocEnumpyEreferenceEc-api.html