GeoPython 随笔：空间数据查询——测试代码

#!/usr/bin/python
import sys
import subprocess
import os
import commands
from datetime import datetime
from datetime import timedelta
import time
import re
import math
import cmd

# This following script aim to do experiments in hadoop.

# ############################################################################################################
#   global variables
# ############################################################################################################
logger = open('./logger.out', 'w')
logger_error = open('./logger.err', 'w')
hadoopDir = ''
samples = []

# ############################################################################################################
#   function execute command
# ############################################################################################################
def execute_command(cmd):
result = executionTime()
logger.write(cmd+'\n')
print cmd
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
logger_error.write(cmd+'\n')
for line in p.stderr.readlines():
logger_error.write(line)
print 'error >>>>> '+ line
print line
for line in p.stdout.readlines():
logger.write(line)
print 'out >>>>>> ' + line
result.parseLines(line)
'''
if 'time in ' in line:
tokens = line.split()
length = len(tokens)
executionTime = tokens[length-1]
'''
logger.write('***->Execution time in Milliseconds(' + result.runTime.replace('\n','')+') ')
print '***->Execution time of ' + result.runTime.replace('\n','')
return result

# ############################################################################################################
#   function close writers
# ############################################################################################################

def close_writers():
logger.flush()
logger_error.flush()
logger.close()
logger_error.close()

def log(message):
logger.write('\n----------\n\t'+message+'\n-----------\n')

# ############################################################################################################
#   Classes
# ############################################################################################################

class executionTime(object):

def __init__(self):
self.runTime = ''
self.sampleTime = ''
self.subdivisionTime = ''

def parseLines(self,line):
if 'Total time for sampling' in line:
token = line.split(" ")
self.sampleTime = token[len(token)-1]
elif 'Total time for space subdivision' in line:
token = line.split(" ")
self.subdivisionTime = token[len(token)-1]
elif 'time in' in line:
token = line.split(" ")
self.runTime = token[len(token)-1]

# ############################################################################################################
#   Upload the test datasets
# ############################################################################################################

datadir = '../testdata/test/'
datasets = ['wcounties','lakes','buildings','allobjects']

def upload():
for dataset in datasets:
cmd = 'bin/hadoop' + ' fs -put ' + hadoopDir + datadir + dataset + ' /user/yaoxx394/'
logger.write(cmd+'\n')
t = execute_command(cmd)
logger.write(','+ str(t))

# ############################################################################################################
#   Experiments on range query
# ############################################################################################################

# ------ function - sample
def read_Sample():
sample_count = 5

#$ bin/shadoop rangequery test.grid rq_results rect:500,500,1000,1000 shape:rect
cmd = 'hadoop jar  05shadoop.jar sample /user/yaoxx394/testdata'+' count:'+ str(sample_count)+' shape:rect outshape:point '
logger.write(cmd+'\n')
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
logger_error.write(cmd)
for line in p.stderr.readlines():
logger_error.write(line)
print 'error >>>>> '+ line
list = re.findall( "\A[(-/+)0-9]+\.[0-9]+,[(-/+)0-9]+\.[0-9]+", line)
if list:
samples.append(list)
for line in p.stdout.readlines():
logger.write(line)
print 'out >>>>>> ' + line
list = re.findall( "\A[(-/+)0-9]+\.[0-9]+,[(-/+)0-9]+\.[0-9]+", line)
if list:
samples.append(list)
return samples

# get total area
def getTotalArea(datafile):
-180,-90,180,90
xmax = 180
xmin = -180
ymax = 90
ymin = -90
area = (xmax-xmin)*(ymax-ymin)
return area

# --- function return the rectangle of center point
def get_rectangle(point,ratio,datafile):
area_ratio = ratio
token = point[0].split(",")
x = float(token[0])
y = float(token[1])
l = math.sqrt(area_ratio * getTotalArea(datafile))
max_x = x+l
max_y = y+l
return str(x)+','+str(y)+','+str(max_x)+','+str(max_y)

def get_mbr(rectangle_list):
mbr = ''
print '**** range query'
for itmes in rectangle_list:
mbr += ' rect:' + itmes
return mbr

def execute_range_query():
list_point = read_Sample()
rectangles = []
dataFile = ""

for point in list_point:
rectangles.append(get_rectangle(point,0.001,dataFile))
mbr = get_mbr(rectangles)
cmd = 'hadoop jar 05shadoop.jar  rangequery  wc.index  allqu '+mbr+' shape:shppolygon -overwrite'
executionTime = execute_command(cmd)
#logger.write(cmd+'\n')
# executionTime = ''
#p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
#logger_error.write(cmd)
#for line in p.stderr.readlines():
#   logger_error.write(line)
#  print 'error >>>>> '+ line
#for line in p.stdout.readlines():
#   logger.write(line)
#  print 'out >>>>>> ' + line
# if 'Time' in line:
#   tokens = line.split()
#  length = len(tokens)
# executionTime = tokens[length-2]
# logger.write('***->Execution time in Milliseconds(' + executionTime+') ')
#print '***->Execution time of range query in millseconds '+ executionTime
logger.write(','+ str(executionTime))
#print '***->Execution time of range query in millseconds '+ executionTime

###############################################################################################
#Generate Synth Data #
################################################################################################

def generateSynth_dataSets():
log(' Genera# get total area Synth Data ')
shape = 'rect'
rect = '-180,-90,180,90'

rectsSize = '100.mb'
t = 0
cmd = 'hadoop jar  05shadoop.jar generate /user/yaoxx394/testdata  mbr:' + rect+ ' size:' + rectsSize+' shape:' + shape +" -overwrite"
t = execute_command(cmd)
return t

# ###############################################################################################################
#   Main()
# ###############################################################################################################
generateSynth_dataSets()