@lupnfer
2016-12-19T13:17:03.000000Z
字数 16714
阅读 779
工作
.
├── build_opensource.py F主编译脚本
├── build_scripts D子库编译脚本
│ ├── build_boost.py
│ ├── build_curl.py
│ ├── build_gflags.py
│ ├── build_glog.py
│ ├── build_googletest.py
│ ├── build_gperftools.py
│ ├── build_jsoncpp.py
│ ├── build_libevent.py
│ ├── build_libodb_boost.py
│ ├── build_libodb_pgsql.py
│ ├── build_libodb.py
│ ├── build_libodb_sqlite.py
│ ├── build_libpq.py
│ ├── build_openssl.py
│ ├── build_protobuf.py
│ ├── build_sqlite.py
│ ├── build_thrift.py
│ ├── build_zeromq.py
│ ├── build_zlib.py
│ └── common.py F基础函数
├── makefiles
│ ├── boost-common
│ ├── boost-cygwin
│ ├── boost-linux
│ ├── boost-windows
│ ├── libevent-windows
│ ├── libodb-windows
│ ├── sqlite-windows
│ ├── thrift-cpp-windows
│ ├── zeromq-windows
│ └── zlib-windows
├── packages D基础库第三方库源
│ ├── boost_1_61_0.tar.bz2
│ ├── curl-7.46.0.tar.bz2
│ ├── gflags
│ ├── glog
│ ├── googletest
│ ├── gperftools
│ ├── jsoncpp
│ ├── libevent-2.0.22-stable.tar.gz
│ ├── libodb-2.4.0.tar.bz2
│ ├── libodb-boost-2.4.0.tar.bz2
│ ├── libodb-pgsql-2.4.0.tar.bz2
│ ├── libodb-sqlite-2.4.0.tar.bz2
│ ├── openssl-1.0.2h.tar.gz
│ ├── postgresql-9.4.5.tar.bz2
│ ├── protobuf-cpp-3.1.0.tar.gz
│ ├── sqlite-autoconf-3100200.tar.gz
│ ├── thrift-0.9.3.tar.gz
│ ├── zeromq-4.1.3.tar.gz
│ └── zlib-1.2.8.tar.gz
├── README.md
└── source_include
└── boost
#!/usr/bin/env python
#encoding: utf-8
#
# opensource build scripts for IA10K project
# Author: jibin 00081
#
import os
import sys
import glob
import shutil
import tarfile
import platform
import subprocess
import logging
import time
import re
import optparse
sys.path.append(os.path.join(os.path.dirname(__file__), 'build_scripts'))
from common import *
if __name__ == '__main__':
parser = optparse.OptionParser()
parser.add_option('-D', '--debug',
help='show debug when build.',
dest='debug',
action='store_true',
default=False)
parser.add_option('-q', '--quiet',
help='not show verbose when build.',
dest='verbose',
action='store_false',
default=True)
parser.add_option('-f', '--force',
help='force to rebuild.',
dest='force',
action='store_true',
default=False)
parser.add_option('-c', '--compiler',
help='compiler to build. valid: %s' % ','.join(get_compiler_list()),
dest='compiler',
default=get_default_compiler())
parser.add_option('-p', '--package',
help='module to build. valid: %s' % ','.join(get_package_list()),
dest='package',
default=get_package_default())
parser.add_option('-a', '--arch',
help='arch to build. valid: %s' % ','.join(get_arch_list()),
dest='arch',
default=get_arch_default())
parser.add_option('-t', '--target',
help='target to build. valid: %s' % ','.join(get_target_list()),
dest='target',
default=get_target_default())
parser.add_option('-m', '--no-yum',
help='compile all source, default is using yum devel package instead of compile it self',
dest='yum',
action='store_false',
default=True)
(options, args) = parser.parse_args()
# set logging level
logging_level = logging.WARN
if options.verbose:
logging_level=logging.INFO
if options.debug:
logging_level=logging.DEBUG
logging.basicConfig(
level=logging_level,
format='[%(asctime)s] [%(filename)s:%(lineno)d] [%(levelname)s] %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
)
c = options.compiler
ps = options.package.split(',')
ar = options.arch.split(',')
ts = options.target.split(',')
if options.package == 'all':
ps = get_package_list()
if options.arch == 'all':
ar = get_arch_list()
if options.target == 'all':
ts = get_target_list()
for p in ps:
for a in ar:
for t in ts:
logging.info('build %s %s %s using %s', p, a, t, c)
bc = BuilderFactory(p)
b = bc(a, t, c, options.force)
b.verbose = options.verbose
b.debug = options.debug
b.yum = options.yum
try:
b.build()
except Exception, ex:
logging.error('build %s %s %s using %s failed.', p, a, t, c)
import traceback
traceback.print_exc(ex)
sys.exit(1)
#!/usr/bin/env python
#encoding: utf-8
import sys
import os
import glob
import logging
import shutil
import time
import subprocess
import re
import multiprocessing
_vc_lists_ = ['vc14', 'vc12', 'vc11', 'vc10']
def get_vcvarsall(ver):
"""
get where vc is installed, return the full-path of vcvarsall.bat
"""
try:
import _winreg
rp = ''
if sys.maxsize > 2**32:
rp = 'Software\Wow6432Node\Microsoft\VisualStudio\%.1f\Setup\VC' % float(ver)
else:
rp = 'Software\Microsoft\VisualStudio\%.1f\Setup\VC' % float(ver)
key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, rp)
vs = _winreg.QueryValueEx(key, 'ProductDir')
if os.path.isfile(os.path.join(vs[0], 'vcvarsall.bat')):
return os.path.join(vs[0], 'vcvarsall.bat')
except Exception, ex:
pass
return None
def get_gcc_march():
return 'core2'
def get_compiler_list():
if sys.platform.startswith('linux'):
return ['gcc']
elif sys.platform == 'cygwin':
return ['gcc']
elif sys.platform == 'win32':
return _vc_lists_
def get_default_compiler():
if sys.platform.startswith('linux'):
return 'gcc'
elif sys.platform == 'cygwin':
return 'gcc'
elif sys.platform == 'win32':
for vc in _vc_lists_:
ver = vc.replace('vc', '')
if get_vcvarsall(ver):
return vc
return 'NONE'
def get_package_list():
ret = []
for x in os.listdir(os.path.dirname(__file__)):
if x.endswith('.py') and x.startswith('build_'):
mx = x.replace('.py', '').replace('build_', '')
ret.append(mx)
return ret
def get_package_default():
return ','.join(get_package_list())
def get_target_list():
return ['Debug', 'RelWithDebInfo']
def get_target_default():
return 'RelWithDebInfo'
def get_arch_list():
if sys.platform == 'cygwin':
return ['x64']
else:
return ['x86', 'x64']
def get_arch_default():
return 'x64'
def patch_textfile_(filename, pattern, replace):
content = None
with open(filename) as fp:
content = fp.read()
if content:
content = re.sub(pattern, replace, content)
with open(filename, 'w') as fp:
fp.write(content)
def cleanup_build_(build_parent, pattern):
for x in glob.glob(os.path.join(build_parent, pattern)):
logging.info('cleanup deleting %s' % x)
try:
shutil.rmtree(x)
except Exception:
pass
time.sleep(1)
def BuilderFactory(package):
try:
exec('from build_%s import %sBuilder' % (package, package))
exec('x=%sBuilder' % package)
return x
except Exception:
logging.error('no class %sBuilder found' % package)
return None
class BasicBuilder():
def __init__(self, package, arch, target, compiler, force):
self.package = package
self.arch = arch
self.target = target
self.compiler = compiler
self.force = force
self.build_parent = os.path.realpath(os.path.join('build', compiler, arch, target))
self.build_touch = os.path.join(self.build_parent, 'buildstamp_%s' % package)
self.windows = False
self.linux = False
self.cygwin = False
self.vc_ver = 0
self.verbose = True
self.debug = False
self.cpu = multiprocessing.cpu_count()
if self.compiler.startswith('vc'):
self.vc_ver = int(self.compiler.replace('vc', ''))
self.platform = ''
if sys.platform.startswith('linux'):
self.linux = True
self.platform = 'linux'
os.environ['BUILD_TOOL_CMAKE'] = os.path.realpath('../tools/cmake/bin/cmake')
elif sys.platform == 'cygwin':
self.cygwin = True
self.platform = 'cygwin'
os.environ['BUILD_TOOL_CMAKE'] = 'cmake'
elif sys.platform == 'win32':
self.windows = True
self.platform = 'windows'
os.environ['BUILD_TOOL_CMAKE'] = os.path.realpath('../tools/cmake/bin/cmake.exe')
self.install_dir = os.path.realpath(os.path.join('dest', self.platform, arch, compiler, target))
self.install_inc_dir = os.path.join(self.install_dir, 'include')
self.install_lib_dir = os.path.join(self.install_dir, 'lib')
if self.linux or self.cygwin:
os.environ['PKG_CONFIG_PATH'] = os.path.join(self.install_lib_dir, 'pkgconfig')
def depends(self):
raise Exception('not impl!')
def build(self):
if self.package not in get_package_list():
logging.warn('package %s is not valid', self.package)
return
if self.arch not in get_arch_list():
logging.warn('arch %s is not valid', sefl.arch)
return
if self.target not in get_target_list():
logging.warn('target %s is not valid', self.target)
return
# do build depends
for p in self.depends():
BC = BuilderFactory(p)
pb = BC(self.arch, self.target, self.compiler, False)
pb.verbose = self.verbose
pb.debug = self.debug
pb.yum = self.yum
pb.build()
do_build = True
if not self.force:
if os.path.isfile(self.build_touch):
st0 = os.stat(self.build_touch)
mts = []
# mts.append(os.stat(__file__).st_mtime)
# mts.append(os.stat(sys.argv[0]).st_mtime)
mts.append(os.stat(os.path.join('build_scripts', 'build_%s.py' % self.package)).st_mtime)
for p in self.depends():
ptouch = os.path.join(self.build_parent, 'buildstamp_%s' % p)
mts.append(os.stat(ptouch).st_mtime)
do_build = False
for mt in mts:
if st0.st_mtime < mt:
do_build = True
break
if do_build:
if not os.path.isdir(self.build_parent):
os.makedirs(self.build_parent)
oldllp = None
if os.environ.has_key('LD_LIBRARY_PATH'):
oldllp = os.environ['LD_LIBRARY_PATH']
oldpath = None
if os.environ.has_key('PATH'):
oldpath = os.environ['PATH']
os.environ['LD_LIBRARY_PATH'] = os.path.join(self.install_lib_dir)
os.environ['PATH'] = '%s:%s' % (os.path.join(self.install_dir, 'bin'), oldpath)
self.prebuild()
self.compile()
self.install()
self.postbuild()
del os.environ['LD_LIBRARY_PATH']
if oldllp:
os.environ['LD_LIBRARY_PATH'] = oldllp
del os.environ['PATH']
if oldpath:
os.environ['PATH'] = oldpath
with open(self.build_touch, 'w') as fp:
fp.write('ok')
else:
logging.warn('build %s %s %s using %s skipped.', self.package, self.arch, self.target, self.compiler)
def prebuild(self):
raise Exception('not impl')
def compile(self):
raise Exception('not impl')
def install(self):
raise Exception('not impl')
def postbuild(self):
raise Exception('not impl')
def find_build_(self, pattern):
fd = glob.glob(os.path.join(self.build_parent, pattern))
if len(fd) != 1 or (not os.path.isdir(fd[0])):
return None
else:
logging.info('find build dir %s', fd[0])
return fd[0]
def find_package_(self, pattern):
fs = glob.glob(os.path.join('packages', pattern))
if len(fs) != 1:
raise Exception('unexpected source packages got: %s', fs)
return fs[0]
def extract_(self, pattern, dest_pattern):
fs = glob.glob(os.path.join('packages', pattern))
if len(fs) != 1:
raise Exception('unexpected source packages got: %s', fs)
logging.info('extracting source %s -> %s', fs[0], self.build_parent)
tool7za = None
if self.cygwin or self.windows:
tool7za = os.path.realpath('../tools/7z/7za.exe')
bp = self.build_parent
np = fs[0]
if self.cygwin:
# translate to windows path
fp = subprocess.Popen(['cygpath', '-d', self.build_parent], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
bp = fp.stdout.read().strip().replace('\\', '/')
fp = subprocess.Popen(['cygpath', '-d', fs[0]], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
np = fp.stdout.read().strip().replace('\\', '/')
if self.linux:
if np.endswith('tar') or np.endswith('tar.gz') or np.endswith('tar.bz2'):
os.system('tar xf %s -C %s' % (np, bp))
else:
if np.endswith('tar.gz') or np.endswith('tar.bz2'):
os.system('%s x -mmt -so %s | %s x -si -aoa -ttar -o%s' % (tool7za, np, tool7za, bp))
elif np.endswith('tar') or np.endswith('zip'):
os.system('%s x -mmt -aoa -o%s %s' % (tool7za, bp, np))
fd = glob.glob(os.path.join(self.build_parent, dest_pattern))
if len(fd) != 1 or (not os.path.isdir(fd[0])):
raise Exception('unexpected source build dirs got: %s', fd)
logging.info('extract success %s -> %s', fs[0], fd[0])
return fd[0]
def cleanup_build_(self, pattern):
for x in glob.glob(os.path.join(self.build_parent, pattern)):
logging.info('cleanup deleting %s' % x)
try:
shutil.rmtree(x)
except Exception:
pass
time.sleep(1)
def install_(self, dest_dir, patterns):
"""
install files from build_dir -> dest_dir, match patterns(using glob)
pattern is glob based, and base dir is build dir
"""
if type(patterns) == type(''):
patterns = [patterns]
for pattern in patterns:
fs = glob.glob(os.path.join(self.build_dir, pattern))
for f in fs:
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
if os.path.isfile(f):
logging.info('install %s -> %s', f, dest_dir)
dest_file = os.path.join(dest_dir, os.path.basename(f))
try:
os.unlink(dest_file)
except Exception:
pass
if os.path.islink(f):
os.symlink(os.readlink(f), dest_file)
else:
shutil.copy2(f, dest_dir)
elif os.path.isdir(f):
logging.info('install %s -> %s', f, dest_dir)
d2 = os.path.join(dest_dir, os.path.basename(f))
if os.path.isdir(d2):
# need to remove first
logging.info('delete %s', d2)
shutil.rmtree(d2)
logging.info('copy %s -> %s', f, dest_dir)
shutil.copytree(f, d2)
def get_cmake_generator_(self):
gtable = {
14: "Visual Studio 14 2015",
12: "Visual Studio 12 2013",
11: "Visual Studio 11 2012",
10: "Visual Studio 10 2010",
9: "Visual Studio 9 2008",
}
if self.windows:
generator = gtable[self.vc_ver]
if self.arch.find('64') >= 0:
generator += ' Win64'
return generator
elif self.linux or self.cygwin:
return 'Unix Makefiles'
def cmake_(self, defines={}):
old_pwd = os.getcwd()
os.chdir(self.build_dir)
throw_ex = None
try:
cmake_cmd = ['cmake', '-G', self.get_cmake_generator_()]
if os.environ.has_key('BUILD_TOOL_CMAKE'):
cmake_cmd = [os.environ['BUILD_TOOL_CMAKE'], '-G', self.get_cmake_generator_()]
for d in defines:
cmake_cmd.append('-D')
cmake_cmd.append('%s=%s' % (d, defines[d]))
cmake_cmd.append('.')
logging.info('starting cmake using: %s', cmake_cmd)
fd = None
if self.debug or self.verbose:
fd = subprocess.Popen(cmake_cmd)
else:
fd = subprocess.Popen(cmake_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
fd.wait()
if fd.returncode != 0:
raise Exception('cmake return failed, ret=%d' % fd.returncode)
except Exception, ex:
throw_ex = ex
finally:
os.chdir(old_pwd)
if throw_ex:
raise throw_ex
logging.info('cmake success')
def msbuild_(self, sln, proj):
vcvarsall = get_vcvarsall(self.vc_ver)
if not vcvarsall:
raise Exception('vc%d is not found' % vc_ver)
old_pwd = os.getcwd()
os.chdir(self.build_dir)
throw_ex = None
try:
cmd = 'msbuild %s' % sln
cmd += ' /p:Configuration=%s' % self.target
if proj:
cmd += ' /t:%s' % proj
logging.info('starting msbuild using: %s', cmd)
fd = None
if self.debug or self.verbose:
fd = subprocess.Popen('"%s" x86 & %s' % (vcvarsall, cmd))
else:
fd = subprocess.Popen('"%s" x86 & %s' % (vcvarsall, cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
fd.wait()
if fd.returncode != 0:
raise Exception('msbuild return failed, ret=%d' % fd.returncode)
except Exception, ex:
throw_ex = ex
finally:
os.chdir(old_pwd)
if throw_ex:
raise throw_ex
logging.info('msbuild success')
def build_shell_(self, cmd):
old_pwd = os.getcwd()
os.chdir(self.build_dir)
throw_ex = None
try:
logging.info('starting run: %s', cmd)
fd = None
if self.windows:
if self.debug or self.verbose:
fd = subprocess.Popen('cmd.exe /c %s' % cmd)
else:
fd = subprocess.Popen('cmd.exe /c %s' % cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
elif self.linux or self.cygwin:
if self.debug or self.verbose:
fd = subprocess.Popen(['bash', '-c', '%s' % cmd])
else:
fd = subprocess.Popen(['bash', '-c', '%s' % cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
fd.wait()
if fd.returncode != 0:
raise Exception('run cmd failed, ret=%d' % fd.returncode)
except Exception, ex:
throw_ex = ex
finally:
os.chdir(old_pwd)
if throw_ex:
raise throw_ex
def nmake_(self, makefile=None, options=None, target=None):
"""
call nmake to build on windows system
makefile: makefile filename, if not given, using default(Makefile)
options: make options
target: build target, if not given, using default.
"""
arch = 'x86'
if self.arch.find('64') >= 0:
arch = 'amd64'
vcvarsall = get_vcvarsall(self.vc_ver)
if not vcvarsall:
raise Exception('vc%d is not found' % self.vc_ver)
old_pwd = os.getcwd()
os.chdir(self.build_dir)
throw_ex = None
try:
make_cmd = 'nmake /nologo'
if makefile:
make_cmd += ' /f %s' % makefile
if options:
for o in options:
make_cmd += ' %s' % o
if target:
make_cmd += ' %s' % target
logging.info('starting nmake using: %s', make_cmd)
fd = subprocess.Popen('"%s" %s & %s' % (vcvarsall, self.arch, make_cmd))
fd.wait()
if fd.returncode != 0:
raise Exception('nmake return failed, ret=%d' % fd.returncode)
except Exception, ex:
throw_ex = ex
finally:
os.chdir(old_pwd)
if throw_ex:
raise throw_ex
logging.info('nmake success')
#!/usr/bin/env python
#encoding: utf-8
from common import *
class curlBuilder(BasicBuilder):
def __init__(self, arch, target, compiler, force):
BasicBuilder.__init__(self, 'curl', arch, target, compiler, force)
def depends(self):
return ['zlib', 'openssl']
def prebuild(self):
if self.yum:
return
self.cleanup_build_('curl-*')
self.build_dir = self.extract_('curl-*.tar.*', 'curl-*')
options = {}
options['BUILD_CURL_EXE'] = 'OFF'
options['BUILD_CURL_TESTS'] = 'OFF'
options['CURL_DISABLE_GOPHER'] = 'ON'
options['CURL_DISABLE_IMAP'] = 'ON'
options['CURL_DISABLE_LDAP'] = 'ON'
options['CURL_DISABLE_LDAPS'] = 'ON'
options['CURL_DISABLE_POP3'] = 'ON'
options['CURL_DISABLE_RTSP'] = 'ON'
options['CURL_DISABLE_SMTP'] = 'ON'
options['CURL_DISABLE_TELNET'] = 'ON'
options['CURL_DISABLE_TFTP'] = 'ON'
options['CURL_STATICLIB'] = 'ON'
options['ENABLE_THREADED_RESOLVER'] = 'ON'
options['ENABLE_MANUAL'] = 'OFF'
options['CMAKE_BUILD_TYPE'] = 'RelWithDebInfo'
if self.target == 'Debug':
options['CMAKE_BUILD_TYPE'] = 'Debug'
options['OPENSSL_INCLUDE_DIR'] = self.install_inc_dir
options['ZLIB_INCLUDE_DIR'] = self.install_inc_dir
if self.windows:
options['OPENSSL_CRYPTO_LIBRARY'] = os.path.join(self.install_lib_dir, 'libeay32.lib')
options['OPENSSL_SSL_LIBRARY'] = os.path.join(self.install_lib_dir, 'ssleay32.lib')
options['ZLIB_LIBRARY'] = os.path.join(self.install_lib_dir, 'zlib.lib')
elif self.linux or self.cygwin:
options['OPENSSL_CRYPTO_LIBRARY'] = os.path.join(self.install_lib_dir, 'libcrypto.a')
options['OPENSSL_SSL_LIBRARY'] = os.path.join(self.install_lib_dir, 'libssl.a')
options['ZLIB_LIBRARY'] = os.path.join(self.install_lib_dir, 'libz.a')
if self.arch == 'x86':
options['CMAKE_C_FLAGS'] = '-m32'
else:
options['CMAKE_C_FLAGS'] = '-m64'
options['CMAKE_C_FLAGS'] += ' -march=%s -g' % get_gcc_march()
options['CMAKE_INSTALL_PREFIX'] = self.install_dir
# cmake generate
self.cmake_(defines=options)
if self.windows:
# patch using /Z7 for debug
patch_textfile_(
os.path.join(self.build_dir, 'lib', 'libcurl.vcxproj'),
r'<DebugInformationFormat>.+</DebugInformationFormat>',
r'<DebugInformationFormat>OldStyle</DebugInformationFormat>'
)
def compile(self):
if self.yum:
return
if self.windows:
self.msbuild_('CURL.sln', 'libcurl')
elif self.linux or self.cygwin:
self.build_shell_('make -j %d' % self.cpu)
def install(self):
if self.yum:
return
if self.windows:
self.install_(self.install_lib_dir, 'lib\\%s\\*.lib' % self.target)
self.install_(os.path.join(self.install_inc_dir, 'curl'), 'include\\curl\\*.h')
elif self.linux or self.cygwin:
self.build_shell_('make install')
def postbuild(self):
if self.yum:
os.system('yum -y install libcurl libcurl-devel')