@lupnfer
2016-12-19T13:17:03.000000Z
字数 16714
阅读 843
工作
.├── build_opensource.py F主编译脚本├── build_scripts D子库编译脚本│ ├── build_boost.py│ ├── build_curl.py│ ├── build_gflags.py│ ├── build_glog.py│ ├── build_googletest.py│ ├── build_gperftools.py│ ├── build_jsoncpp.py│ ├── build_libevent.py│ ├── build_libodb_boost.py│ ├── build_libodb_pgsql.py│ ├── build_libodb.py│ ├── build_libodb_sqlite.py│ ├── build_libpq.py│ ├── build_openssl.py│ ├── build_protobuf.py│ ├── build_sqlite.py│ ├── build_thrift.py│ ├── build_zeromq.py│ ├── build_zlib.py│ └── common.py F基础函数├── makefiles│ ├── boost-common│ ├── boost-cygwin│ ├── boost-linux│ ├── boost-windows│ ├── libevent-windows│ ├── libodb-windows│ ├── sqlite-windows│ ├── thrift-cpp-windows│ ├── zeromq-windows│ └── zlib-windows├── packages D基础库第三方库源│ ├── boost_1_61_0.tar.bz2│ ├── curl-7.46.0.tar.bz2│ ├── gflags│ ├── glog│ ├── googletest│ ├── gperftools│ ├── jsoncpp│ ├── libevent-2.0.22-stable.tar.gz│ ├── libodb-2.4.0.tar.bz2│ ├── libodb-boost-2.4.0.tar.bz2│ ├── libodb-pgsql-2.4.0.tar.bz2│ ├── libodb-sqlite-2.4.0.tar.bz2│ ├── openssl-1.0.2h.tar.gz│ ├── postgresql-9.4.5.tar.bz2│ ├── protobuf-cpp-3.1.0.tar.gz│ ├── sqlite-autoconf-3100200.tar.gz│ ├── thrift-0.9.3.tar.gz│ ├── zeromq-4.1.3.tar.gz│ └── zlib-1.2.8.tar.gz├── README.md└── source_include└── boost
#!/usr/bin/env python#encoding: utf-8## opensource build scripts for IA10K project# Author: jibin 00081#import osimport sysimport globimport shutilimport tarfileimport platformimport subprocessimport loggingimport timeimport reimport optparsesys.path.append(os.path.join(os.path.dirname(__file__), 'build_scripts'))from common import *if __name__ == '__main__':parser = optparse.OptionParser()parser.add_option('-D', '--debug',help='show debug when build.',dest='debug',action='store_true',default=False)parser.add_option('-q', '--quiet',help='not show verbose when build.',dest='verbose',action='store_false',default=True)parser.add_option('-f', '--force',help='force to rebuild.',dest='force',action='store_true',default=False)parser.add_option('-c', '--compiler',help='compiler to build. valid: %s' % ','.join(get_compiler_list()),dest='compiler',default=get_default_compiler())parser.add_option('-p', '--package',help='module to build. valid: %s' % ','.join(get_package_list()),dest='package',default=get_package_default())parser.add_option('-a', '--arch',help='arch to build. valid: %s' % ','.join(get_arch_list()),dest='arch',default=get_arch_default())parser.add_option('-t', '--target',help='target to build. valid: %s' % ','.join(get_target_list()),dest='target',default=get_target_default())parser.add_option('-m', '--no-yum',help='compile all source, default is using yum devel package instead of compile it self',dest='yum',action='store_false',default=True)(options, args) = parser.parse_args()# set logging levellogging_level = logging.WARNif options.verbose:logging_level=logging.INFOif options.debug:logging_level=logging.DEBUGlogging.basicConfig(level=logging_level,format='[%(asctime)s] [%(filename)s:%(lineno)d] [%(levelname)s] %(message)s',datefmt='%Y-%m-%d %H:%M:%S',)c = options.compilerps = options.package.split(',')ar = options.arch.split(',')ts = options.target.split(',')if options.package == 'all':ps = get_package_list()if options.arch == 'all':ar = get_arch_list()if options.target == 'all':ts = get_target_list()for p in ps:for a in ar:for t in ts:logging.info('build %s %s %s using %s', p, a, t, c)bc = BuilderFactory(p)b = bc(a, t, c, options.force)b.verbose = options.verboseb.debug = options.debugb.yum = options.yumtry:b.build()except Exception, ex:logging.error('build %s %s %s using %s failed.', p, a, t, c)import tracebacktraceback.print_exc(ex)sys.exit(1)
#!/usr/bin/env python#encoding: utf-8import sysimport osimport globimport loggingimport shutilimport timeimport subprocessimport reimport multiprocessing_vc_lists_ = ['vc14', 'vc12', 'vc11', 'vc10']def get_vcvarsall(ver):"""get where vc is installed, return the full-path of vcvarsall.bat"""try:import _winregrp = ''if sys.maxsize > 2**32:rp = 'Software\Wow6432Node\Microsoft\VisualStudio\%.1f\Setup\VC' % float(ver)else:rp = 'Software\Microsoft\VisualStudio\%.1f\Setup\VC' % float(ver)key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, rp)vs = _winreg.QueryValueEx(key, 'ProductDir')if os.path.isfile(os.path.join(vs[0], 'vcvarsall.bat')):return os.path.join(vs[0], 'vcvarsall.bat')except Exception, ex:passreturn Nonedef get_gcc_march():return 'core2'def get_compiler_list():if sys.platform.startswith('linux'):return ['gcc']elif sys.platform == 'cygwin':return ['gcc']elif sys.platform == 'win32':return _vc_lists_def get_default_compiler():if sys.platform.startswith('linux'):return 'gcc'elif sys.platform == 'cygwin':return 'gcc'elif sys.platform == 'win32':for vc in _vc_lists_:ver = vc.replace('vc', '')if get_vcvarsall(ver):return vcreturn 'NONE'def get_package_list():ret = []for x in os.listdir(os.path.dirname(__file__)):if x.endswith('.py') and x.startswith('build_'):mx = x.replace('.py', '').replace('build_', '')ret.append(mx)return retdef get_package_default():return ','.join(get_package_list())def get_target_list():return ['Debug', 'RelWithDebInfo']def get_target_default():return 'RelWithDebInfo'def get_arch_list():if sys.platform == 'cygwin':return ['x64']else:return ['x86', 'x64']def get_arch_default():return 'x64'def patch_textfile_(filename, pattern, replace):content = Nonewith open(filename) as fp:content = fp.read()if content:content = re.sub(pattern, replace, content)with open(filename, 'w') as fp:fp.write(content)def cleanup_build_(build_parent, pattern):for x in glob.glob(os.path.join(build_parent, pattern)):logging.info('cleanup deleting %s' % x)try:shutil.rmtree(x)except Exception:passtime.sleep(1)def BuilderFactory(package):try:exec('from build_%s import %sBuilder' % (package, package))exec('x=%sBuilder' % package)return xexcept Exception:logging.error('no class %sBuilder found' % package)return Noneclass BasicBuilder():def __init__(self, package, arch, target, compiler, force):self.package = packageself.arch = archself.target = targetself.compiler = compilerself.force = forceself.build_parent = os.path.realpath(os.path.join('build', compiler, arch, target))self.build_touch = os.path.join(self.build_parent, 'buildstamp_%s' % package)self.windows = Falseself.linux = Falseself.cygwin = Falseself.vc_ver = 0self.verbose = Trueself.debug = Falseself.cpu = multiprocessing.cpu_count()if self.compiler.startswith('vc'):self.vc_ver = int(self.compiler.replace('vc', ''))self.platform = ''if sys.platform.startswith('linux'):self.linux = Trueself.platform = 'linux'os.environ['BUILD_TOOL_CMAKE'] = os.path.realpath('../tools/cmake/bin/cmake')elif sys.platform == 'cygwin':self.cygwin = Trueself.platform = 'cygwin'os.environ['BUILD_TOOL_CMAKE'] = 'cmake'elif sys.platform == 'win32':self.windows = Trueself.platform = 'windows'os.environ['BUILD_TOOL_CMAKE'] = os.path.realpath('../tools/cmake/bin/cmake.exe')self.install_dir = os.path.realpath(os.path.join('dest', self.platform, arch, compiler, target))self.install_inc_dir = os.path.join(self.install_dir, 'include')self.install_lib_dir = os.path.join(self.install_dir, 'lib')if self.linux or self.cygwin:os.environ['PKG_CONFIG_PATH'] = os.path.join(self.install_lib_dir, 'pkgconfig')def depends(self):raise Exception('not impl!')def build(self):if self.package not in get_package_list():logging.warn('package %s is not valid', self.package)returnif self.arch not in get_arch_list():logging.warn('arch %s is not valid', sefl.arch)returnif self.target not in get_target_list():logging.warn('target %s is not valid', self.target)return# do build dependsfor p in self.depends():BC = BuilderFactory(p)pb = BC(self.arch, self.target, self.compiler, False)pb.verbose = self.verbosepb.debug = self.debugpb.yum = self.yumpb.build()do_build = Trueif not self.force:if os.path.isfile(self.build_touch):st0 = os.stat(self.build_touch)mts = []# mts.append(os.stat(__file__).st_mtime)# mts.append(os.stat(sys.argv[0]).st_mtime)mts.append(os.stat(os.path.join('build_scripts', 'build_%s.py' % self.package)).st_mtime)for p in self.depends():ptouch = os.path.join(self.build_parent, 'buildstamp_%s' % p)mts.append(os.stat(ptouch).st_mtime)do_build = Falsefor mt in mts:if st0.st_mtime < mt:do_build = Truebreakif do_build:if not os.path.isdir(self.build_parent):os.makedirs(self.build_parent)oldllp = Noneif os.environ.has_key('LD_LIBRARY_PATH'):oldllp = os.environ['LD_LIBRARY_PATH']oldpath = Noneif os.environ.has_key('PATH'):oldpath = os.environ['PATH']os.environ['LD_LIBRARY_PATH'] = os.path.join(self.install_lib_dir)os.environ['PATH'] = '%s:%s' % (os.path.join(self.install_dir, 'bin'), oldpath)self.prebuild()self.compile()self.install()self.postbuild()del os.environ['LD_LIBRARY_PATH']if oldllp:os.environ['LD_LIBRARY_PATH'] = oldllpdel os.environ['PATH']if oldpath:os.environ['PATH'] = oldpathwith open(self.build_touch, 'w') as fp:fp.write('ok')else:logging.warn('build %s %s %s using %s skipped.', self.package, self.arch, self.target, self.compiler)def prebuild(self):raise Exception('not impl')def compile(self):raise Exception('not impl')def install(self):raise Exception('not impl')def postbuild(self):raise Exception('not impl')def find_build_(self, pattern):fd = glob.glob(os.path.join(self.build_parent, pattern))if len(fd) != 1 or (not os.path.isdir(fd[0])):return Noneelse:logging.info('find build dir %s', fd[0])return fd[0]def find_package_(self, pattern):fs = glob.glob(os.path.join('packages', pattern))if len(fs) != 1:raise Exception('unexpected source packages got: %s', fs)return fs[0]def extract_(self, pattern, dest_pattern):fs = glob.glob(os.path.join('packages', pattern))if len(fs) != 1:raise Exception('unexpected source packages got: %s', fs)logging.info('extracting source %s -> %s', fs[0], self.build_parent)tool7za = Noneif self.cygwin or self.windows:tool7za = os.path.realpath('../tools/7z/7za.exe')bp = self.build_parentnp = fs[0]if self.cygwin:# translate to windows pathfp = subprocess.Popen(['cygpath', '-d', self.build_parent], stdout=subprocess.PIPE, stderr=subprocess.PIPE)bp = fp.stdout.read().strip().replace('\\', '/')fp = subprocess.Popen(['cygpath', '-d', fs[0]], stdout=subprocess.PIPE, stderr=subprocess.PIPE)np = fp.stdout.read().strip().replace('\\', '/')if self.linux:if np.endswith('tar') or np.endswith('tar.gz') or np.endswith('tar.bz2'):os.system('tar xf %s -C %s' % (np, bp))else:if np.endswith('tar.gz') or np.endswith('tar.bz2'):os.system('%s x -mmt -so %s | %s x -si -aoa -ttar -o%s' % (tool7za, np, tool7za, bp))elif np.endswith('tar') or np.endswith('zip'):os.system('%s x -mmt -aoa -o%s %s' % (tool7za, bp, np))fd = glob.glob(os.path.join(self.build_parent, dest_pattern))if len(fd) != 1 or (not os.path.isdir(fd[0])):raise Exception('unexpected source build dirs got: %s', fd)logging.info('extract success %s -> %s', fs[0], fd[0])return fd[0]def cleanup_build_(self, pattern):for x in glob.glob(os.path.join(self.build_parent, pattern)):logging.info('cleanup deleting %s' % x)try:shutil.rmtree(x)except Exception:passtime.sleep(1)def install_(self, dest_dir, patterns):"""install files from build_dir -> dest_dir, match patterns(using glob)pattern is glob based, and base dir is build dir"""if type(patterns) == type(''):patterns = [patterns]for pattern in patterns:fs = glob.glob(os.path.join(self.build_dir, pattern))for f in fs:if not os.path.isdir(dest_dir):os.makedirs(dest_dir)if os.path.isfile(f):logging.info('install %s -> %s', f, dest_dir)dest_file = os.path.join(dest_dir, os.path.basename(f))try:os.unlink(dest_file)except Exception:passif os.path.islink(f):os.symlink(os.readlink(f), dest_file)else:shutil.copy2(f, dest_dir)elif os.path.isdir(f):logging.info('install %s -> %s', f, dest_dir)d2 = os.path.join(dest_dir, os.path.basename(f))if os.path.isdir(d2):# need to remove firstlogging.info('delete %s', d2)shutil.rmtree(d2)logging.info('copy %s -> %s', f, dest_dir)shutil.copytree(f, d2)def get_cmake_generator_(self):gtable = {14: "Visual Studio 14 2015",12: "Visual Studio 12 2013",11: "Visual Studio 11 2012",10: "Visual Studio 10 2010",9: "Visual Studio 9 2008",}if self.windows:generator = gtable[self.vc_ver]if self.arch.find('64') >= 0:generator += ' Win64'return generatorelif self.linux or self.cygwin:return 'Unix Makefiles'def cmake_(self, defines={}):old_pwd = os.getcwd()os.chdir(self.build_dir)throw_ex = Nonetry:cmake_cmd = ['cmake', '-G', self.get_cmake_generator_()]if os.environ.has_key('BUILD_TOOL_CMAKE'):cmake_cmd = [os.environ['BUILD_TOOL_CMAKE'], '-G', self.get_cmake_generator_()]for d in defines:cmake_cmd.append('-D')cmake_cmd.append('%s=%s' % (d, defines[d]))cmake_cmd.append('.')logging.info('starting cmake using: %s', cmake_cmd)fd = Noneif self.debug or self.verbose:fd = subprocess.Popen(cmake_cmd)else:fd = subprocess.Popen(cmake_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)fd.wait()if fd.returncode != 0:raise Exception('cmake return failed, ret=%d' % fd.returncode)except Exception, ex:throw_ex = exfinally:os.chdir(old_pwd)if throw_ex:raise throw_exlogging.info('cmake success')def msbuild_(self, sln, proj):vcvarsall = get_vcvarsall(self.vc_ver)if not vcvarsall:raise Exception('vc%d is not found' % vc_ver)old_pwd = os.getcwd()os.chdir(self.build_dir)throw_ex = Nonetry:cmd = 'msbuild %s' % slncmd += ' /p:Configuration=%s' % self.targetif proj:cmd += ' /t:%s' % projlogging.info('starting msbuild using: %s', cmd)fd = Noneif self.debug or self.verbose:fd = subprocess.Popen('"%s" x86 & %s' % (vcvarsall, cmd))else:fd = subprocess.Popen('"%s" x86 & %s' % (vcvarsall, cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE)fd.wait()if fd.returncode != 0:raise Exception('msbuild return failed, ret=%d' % fd.returncode)except Exception, ex:throw_ex = exfinally:os.chdir(old_pwd)if throw_ex:raise throw_exlogging.info('msbuild success')def build_shell_(self, cmd):old_pwd = os.getcwd()os.chdir(self.build_dir)throw_ex = Nonetry:logging.info('starting run: %s', cmd)fd = Noneif self.windows:if self.debug or self.verbose:fd = subprocess.Popen('cmd.exe /c %s' % cmd)else:fd = subprocess.Popen('cmd.exe /c %s' % cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)elif self.linux or self.cygwin:if self.debug or self.verbose:fd = subprocess.Popen(['bash', '-c', '%s' % cmd])else:fd = subprocess.Popen(['bash', '-c', '%s' % cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE)fd.wait()if fd.returncode != 0:raise Exception('run cmd failed, ret=%d' % fd.returncode)except Exception, ex:throw_ex = exfinally:os.chdir(old_pwd)if throw_ex:raise throw_exdef nmake_(self, makefile=None, options=None, target=None):"""call nmake to build on windows systemmakefile: makefile filename, if not given, using default(Makefile)options: make optionstarget: build target, if not given, using default."""arch = 'x86'if self.arch.find('64') >= 0:arch = 'amd64'vcvarsall = get_vcvarsall(self.vc_ver)if not vcvarsall:raise Exception('vc%d is not found' % self.vc_ver)old_pwd = os.getcwd()os.chdir(self.build_dir)throw_ex = Nonetry:make_cmd = 'nmake /nologo'if makefile:make_cmd += ' /f %s' % makefileif options:for o in options:make_cmd += ' %s' % oif target:make_cmd += ' %s' % targetlogging.info('starting nmake using: %s', make_cmd)fd = subprocess.Popen('"%s" %s & %s' % (vcvarsall, self.arch, make_cmd))fd.wait()if fd.returncode != 0:raise Exception('nmake return failed, ret=%d' % fd.returncode)except Exception, ex:throw_ex = exfinally:os.chdir(old_pwd)if throw_ex:raise throw_exlogging.info('nmake success')
#!/usr/bin/env python#encoding: utf-8from common import *class curlBuilder(BasicBuilder):def __init__(self, arch, target, compiler, force):BasicBuilder.__init__(self, 'curl', arch, target, compiler, force)def depends(self):return ['zlib', 'openssl']def prebuild(self):if self.yum:returnself.cleanup_build_('curl-*')self.build_dir = self.extract_('curl-*.tar.*', 'curl-*')options = {}options['BUILD_CURL_EXE'] = 'OFF'options['BUILD_CURL_TESTS'] = 'OFF'options['CURL_DISABLE_GOPHER'] = 'ON'options['CURL_DISABLE_IMAP'] = 'ON'options['CURL_DISABLE_LDAP'] = 'ON'options['CURL_DISABLE_LDAPS'] = 'ON'options['CURL_DISABLE_POP3'] = 'ON'options['CURL_DISABLE_RTSP'] = 'ON'options['CURL_DISABLE_SMTP'] = 'ON'options['CURL_DISABLE_TELNET'] = 'ON'options['CURL_DISABLE_TFTP'] = 'ON'options['CURL_STATICLIB'] = 'ON'options['ENABLE_THREADED_RESOLVER'] = 'ON'options['ENABLE_MANUAL'] = 'OFF'options['CMAKE_BUILD_TYPE'] = 'RelWithDebInfo'if self.target == 'Debug':options['CMAKE_BUILD_TYPE'] = 'Debug'options['OPENSSL_INCLUDE_DIR'] = self.install_inc_diroptions['ZLIB_INCLUDE_DIR'] = self.install_inc_dirif self.windows:options['OPENSSL_CRYPTO_LIBRARY'] = os.path.join(self.install_lib_dir, 'libeay32.lib')options['OPENSSL_SSL_LIBRARY'] = os.path.join(self.install_lib_dir, 'ssleay32.lib')options['ZLIB_LIBRARY'] = os.path.join(self.install_lib_dir, 'zlib.lib')elif self.linux or self.cygwin:options['OPENSSL_CRYPTO_LIBRARY'] = os.path.join(self.install_lib_dir, 'libcrypto.a')options['OPENSSL_SSL_LIBRARY'] = os.path.join(self.install_lib_dir, 'libssl.a')options['ZLIB_LIBRARY'] = os.path.join(self.install_lib_dir, 'libz.a')if self.arch == 'x86':options['CMAKE_C_FLAGS'] = '-m32'else:options['CMAKE_C_FLAGS'] = '-m64'options['CMAKE_C_FLAGS'] += ' -march=%s -g' % get_gcc_march()options['CMAKE_INSTALL_PREFIX'] = self.install_dir# cmake generateself.cmake_(defines=options)if self.windows:# patch using /Z7 for debugpatch_textfile_(os.path.join(self.build_dir, 'lib', 'libcurl.vcxproj'),r'<DebugInformationFormat>.+</DebugInformationFormat>',r'<DebugInformationFormat>OldStyle</DebugInformationFormat>')def compile(self):if self.yum:returnif self.windows:self.msbuild_('CURL.sln', 'libcurl')elif self.linux or self.cygwin:self.build_shell_('make -j %d' % self.cpu)def install(self):if self.yum:returnif self.windows:self.install_(self.install_lib_dir, 'lib\\%s\\*.lib' % self.target)self.install_(os.path.join(self.install_inc_dir, 'curl'), 'include\\curl\\*.h')elif self.linux or self.cygwin:self.build_shell_('make install')def postbuild(self):if self.yum:os.system('yum -y install libcurl libcurl-devel')