python手记(20)

python-smp多进程运算

>>> ================================ RESTART ================================

>>> 

Usage: python callback.py [ncpus]

    [ncpus] - the number of workers to run in parallel, 

    if omitted it will be set to the number of processors in the system

Starting pp with 4 workers

Partial sum is 0.69314720556 | diff = -2.5000044479e-08

Job execution statistics:

 job count | % of all jobs | job time sum | time per job | job server

       128 |        100.00 |       7.3840 |     0.057687 | local

Time elapsed since server creation 1.93600010872

0 active tasks, 4 cores

>>> 

使用pp库
http://www.parallelpython.com/
#!/usr/bin/python
# File: sum_primes.py
# Author: VItalii Vanovschi
# Desc: This program demonstrates parallel computations with pp module
# It calculates the sum of prime numbers below a given integer in parallel
# Parallel Python Software: http://www.parallelpython.com
import math, sys, time
import pp

def isprime(n):
"""Returns True if n is prime and False otherwise"""
if not isinstance(n, int):
raise TypeError("argument passed to is_prime is not of 'int' type")
if n < 2:
return False
if n == 2:
return True
max = int(math.ceil(math.sqrt(n)))
i = 2
while i <= max:
if n % i == 0:
return False
i += 1
return True

def sum_primes(n):
"""Calculates sum of all primes below given integer n"""
return sum([x for x in xrange(2,n) if isprime(x)])

print """Usage: python sum_primes.py [ncpus]
[ncpus] - the number of workers to run in parallel,
if omitted it will be set to the number of processors in the system
"""

# tuple of all parallel python servers to connect with
ppservers = ()
#ppservers = ("10.0.0.1",)

if len(sys.argv) > 1:
ncpus = int(sys.argv[1])
# Creates jobserver with ncpus workers
job_server = pp.Server(ncpus, ppservers=ppservers)
else:
# Creates jobserver with automatically detected number of workers
job_server = pp.Server(ppservers=ppservers)

print "Starting pp with", job_server.get_ncpus(), "workers"

# Submit a job of calulating sum_primes(100) for execution.
# sum_primes - the function
# (100,) - tuple with arguments for sum_primes
# (isprime,) - tuple with functions on which function sum_primes depends
# ("math",) - tuple with module names which must be imported before sum_primes execution
# Execution starts as soon as one of the workers will become available
job1 = job_server.submit(sum_primes, (100,), (isprime,), ("math",))

# Retrieves the result calculated by job1
# The value of job1() is the same as sum_primes(100)
# If the job has not been finished yet, execution will wait here until result is available
result = job1()

print "Sum of primes below 100 is", result

start_time = time.time()

# The following submits 8 jobs and then retrieves the results
inputs = (100000, 100100, 100200, 100300, 100400, 100500, 100600, 100700,50001)
jobs = [(input, job_server.submit(sum_primes,(input,), (isprime,), ("math",))) for input in inputs]
for input, job in jobs:
print "Sum of primes below", input, "is", job()

print "Time elapsed: ", time.time() - start_time, "s"
job_server.print_stats()
>>> 

Usage: python sum_primes.py [ncpus]

    [ncpus] - the number of workers to run in parallel, 

    if omitted it will be set to the number of processors in the system

Starting pp with 4 workers

Sum of primes below 100 is 1060

Sum of primes below 100000 is 454396537

Sum of primes below 100100 is 454996777

Sum of primes below 100200 is 455898156

Sum of primes below 100300 is 456700218

Sum of primes below 100400 is 457603451

Sum of primes below 100500 is 458407033

Sum of primes below 100600 is 459412387

Sum of primes below 100700 is 460217613

Sum of primes below 50001 is 121013308

Time elapsed:  1.84899997711 s

Job execution statistics:

 job count | % of all jobs | job time sum | time per job | job server

        10 |        100.00 |       6.6840 |     0.668400 | local

Time elapsed since server creation 1.88800001144

0 active tasks, 4 cores

>>> 

#!/usr/bin/python
# File: callback.py
# Author: Vitalii Vanovschi
# Desc: This program demonstrates parallel computations with pp module
# using callbacks (available since pp 1.3).
# Program calculates the partial sum 1-1/2+1/3-1/4+1/5-1/6+... (in the limit it is ln(2))
# Parallel Python Software: http://www.parallelpython.com 
import math, time, thread, sys
import pp

#class for callbacks
class Sum:
def __init__(self):
self.value = 0.0
self.lock = thread.allocate_lock()
self.count = 0

#the callback function
def add(self, value):
# we must use lock here because += is not atomic
self.count += 1
self.lock.acquire()
self.value += value
self.lock.release()

def part_sum(start, end):
"""Calculates partial sum"""
sum = 0
for x in xrange(start, end):
if x % 2 == 0:
sum -= 1.0 / x
else:
sum += 1.0 / x
return sum

print """Usage: python callback.py [ncpus]
[ncpus] - the number of workers to run in parallel,
if omitted it will be set to the number of processors in the system
"""

start = 1
end = 20000000

# Divide the task into 128 subtasks
parts = 128
step = (end - start) / parts + 1

# tuple of all parallel python servers to connect with
ppservers = ()
#ppservers = ("localhost",)

if len(sys.argv) > 1:
ncpus = int(sys.argv[1])
# Creates jobserver with ncpus workers
job_server = pp.Server(ncpus, ppservers=ppservers)
else:
# Creates jobserver with automatically detected number of workers
job_server = pp.Server(ppservers=ppservers)

print "Starting pp with", job_server.get_ncpus(), "workers"

# Create an instance of callback class
sum = Sum()

# Execute the same task with different amount of active workers and measure the time
start_time = time.time()
for index in xrange(parts):
starti = start+index*step
endi = min(start+(index+1)*step, end)
# Submit a job which will calculate partial sum
# part_sum - the function
# (starti, endi) - tuple with arguments for part_sum
# callback=sum.add - callback function
job_server.submit(part_sum, (starti, endi), callback=sum.add)

#wait for jobs in all groups to finish
job_server.wait()

# Print the partial sum
print "Partial sum is", sum.value, "| diff =", math.log(2) - sum.value

job_server.print_stats()

# Parallel Python Software: http://www.parallelpython.com