From BlenderWiki

Jump to: navigation, search

BuildBot Script

Automatically build and upload on svn changes, can deal with multiple branches and build options.


build_conf.py

config file for the build bot, directories to build and arguments, any changes or additional builds will be detected and built/rebuild while build_bot.py runs

BLENDER_BUILDS = {}
 
trunk = '/home/ideasman42/build/blender'
 
BLENDER_BUILDS['linux_x86-64_lite'] = {\
	'root':trunk, \
	'args':'blenderlite BF_QUIET=1 BF_FANCY=0 WITH_BF_STATICCXX=1 BF_CXX_LIB_STATIC="/usr/lib/gcc/x86_64-linux-gnu/4.3/libstdc++.a" PLATFORM_LINKFLAGS="-pthread -static-libgcc"', \
	'strip':True, \
}
 
BLENDER_BUILDS['linux_x86-64_lite_debug'] = {\
	'root':trunk, \
	'args':BLENDER_BUILDS['linux_x86-64_lite']['args'] + ' BF_DEBUG=1 BF_DEBUG_FLAGS="-O0 -g3 -ggdb3 -fno-inline"', \
	'strip':False, \
}
 
BLENDER_BUILDS['win32_lite'] = {\
	'root':trunk, \
	'args':'blenderlite BF_CROSS=1 WITH_BF_STATICCXX=1  BF_CXX_LIB_STATIC="/home/ideasman42/build/lib/windows/pthreads/lib/libpthreadGC2.a /usr/lib/gcc/i586-mingw32msvc/4.2.1-sjlj/libstdc++.a /home/ideasman42/build/lib/windows/zlib/lib/libz.a /home/ideasman42/build/lib/windows/png/lib/libpng12.a  /home/ideasman42/build/lib/windows/jpeg/lib/libjpeg.a" BF_QUIET=0  BF_PTHREADS_LIB="" BF_PTHREADS_LIBPATH=""', \
	'strip':True, 'strip_cmd':'i586-mingw32msvc-strip',\
	'compress':'zip'
}
 
BLENDER_BUILDS['win32_lite_debug'] = {\
	'root':trunk, \
	'args':BLENDER_BUILDS['win32_lite']['args'] + ' BF_DEBUG=1 BF_DEBUG_FLAGS="-O0 -g3 -ggdb3 -fno-inline"', \
	'strip':False, \
	'compress':'zip'
}
build_bot.py
import os
import shutil # recursive delete
import time
SLEEP = 60
 
WPUT_BIN = '/opt/wput/bin/wput'
WPUT_TARGET = 'ftp://autobuilds@graphicall.org:xxxxxxxx@ftp.graphicall.org/'
WPUT_ARGS= '--dont-continue --reupload'
 
BLENDER_BIN = 'blender'
REBUILD_FULL = True # make clean every time?
VERBOSE = False
 
 
# Better to import so we can hardlink to a chroot
# from build_conf import BLENDER_BUILDS
 
import build_conf
 
def build_rev(ID):
	for l in os.popen('svn info %s' % build_conf.BLENDER_BUILDS[ID]['root']):
		if l.startswith('Last Changed Rev: '):
			return int(l.split()[-1])
	return -1
 
def build_up(ID):
	'''
	return True if we changed
	'''
 
	rev = build_rev(ID)
	if VERBOSE: print 'attempting SVN update, currently at rev:', rev
	os.popen('svn up %s' % build_conf.BLENDER_BUILDS[ID]['root']).read()
	if VERBOSE: print '...done'
	return rev != build_rev(ID)
 
 
 
 
def build_do(ID):
	txt_out = 'build_out_%s.txt' % ID
	txt_warn = 'build_warn_%s.txt' % ID
	dir_build = '../build/linux2_%s' % ID
	dir_install = '../blender-%s' % ID
	file_tar = '../blender_%s.tar' % ID
	file_zip = '../blender_%s.zip' % ID
	file_bin = os.path.join(build_conf.BLENDER_BUILDS[ID]['root'], dir_install, BLENDER_BIN)
 
	# remove old install files
	print '\tblender:', ID, '-- removing all install files'
	if os.path.exists(dir_install):
		shutil.rmtree(dir_install) #os.rmdir( dir_install )
 
	# opposed to a clean build
	if REBUILD_FULL:
		print '\tblender:', ID, '-- clean build...'
		if os.path.exists(dir_build):
			shutil.rmtree(dir_build) #os.rmdir( dir_install )
 
	# Use BF_CONFIG ?
 
	args = build_conf.BLENDER_BUILDS[ID]['args'] + ' '
	args += 'BF_FANCY=0 '
	args += 'BF_BUILDDIR=%s ' % dir_build
	args += 'BF_INSTALLDIR=%s ' % dir_install
 
	# os.system('python scons/scons.py clean')
	print '\tblender:', ID, '-- building blender...'
	os.system('python scons/scons.py %s  1> %s 2> %s' % (args, txt_out, txt_warn) )
	print "\tscons args:", args
	# os.system('python scons/scons.py %s' % args ) # for testing 
 
	# Is this a windows binary?
	if os.path.exists( file_bin + '.exe' ):
		file_bin = file_bin + '.exe'
 
	if not os.path.exists( file_bin ):
		print 'Build Failed! - Not Found:', file_bin
		return
 
	shutil.copyfile(txt_out, os.path.join( dir_install, os.path.basename(txt_out) ))
	shutil.copyfile(txt_warn, os.path.join( dir_install, os.path.basename(txt_warn) ))
 
	# strip the binary for smaller size?
	if build_conf.BLENDER_BUILDS[ID]['strip']:
		if 'strip_cmd' in build_conf.BLENDER_BUILDS[ID]:
			strip_cmd = build_conf.BLENDER_BUILDS[ID]['strip_cmd']
		else:
			strip_cmd = 'strip'
 
		os.system('%s -s "%s"' % (strip_cmd, file_bin))
 
	# Build a list of files to upload
	uploads = []
	'''
	if file_bin.endswith('.exe'): # hack for win32 blenderlite
		os.system('/opt/upx/upx --lzma -9 %s' % file_bin)
		file_bin_new = os.path.basename(file_bin)
		file_bin_new = file_bin_new.replace('blender.exe', 'blender_' + ID + '.exe')
		shutil.move(file_bin, file_bin_new)
		uploads.append(file_bin_new)
	else:
	'''
 
	if 1:
		if 'compress' in build_conf.BLENDER_BUILDS[ID] and build_conf.BLENDER_BUILDS[ID]['compress']=='zip':
			if os.path.exists(file_zip):
				os.remove(file_zip)
 
			cwd = os.getcwd()
			os.chdir(dir_install)
			os.system('zip -9 -r %s *' % (os.path.join(cwd, file_zip)))
			os.chdir(cwd)
			uploads.append(file_zip)
 
		else:
			print '\tblender:', ID, '-- tar files...'
			os.system('tar -cf %s %s' % (file_tar, dir_install))
			print '\tblender:', ID, '-- bzip2 tar files...'
 
			if os.path.exists('%s.bz2' % file_tar):
				os.remove('%s.bz2' % file_tar)
 
			os.system('bzip2 -9 %s' % file_tar)
 
			# TODO - wput
			print '\tblender:', ID, 'uploading'
			print '%s %s.bz2 %s' % (WPUT_BIN, file_tar, WPUT_TARGET)
 
			uploads.append(file_tar + '.bz2')
 
	uploads.append(txt_out)
	uploads.append(txt_warn)
 
	for f in uploads:
		os.system('%s %s %s %s' % (WPUT_BIN, f, WPUT_TARGET, WPUT_ARGS))
 
	print 'done...'
 
 
def build_data_str():
	'''
	Use this to see if build settings change
	'''
	build_data = {}
	for ID, value in build_conf.BLENDER_BUILDS.iteritems():
		build_data[ID] = str(value)
 
	return build_data
 
def main():
	# main build loop
 
	# Allow new builds to be added while the script runs
 
	build_data_prev = build_data_str()
 
	ok = True
	print "Starting build loop for", len(build_conf.BLENDER_BUILDS), 'blenders'
	while ok:
 
		# refresh settings
		reload(build_conf)
		build_data = build_data_str()
 
		# Get unique roots and link with ID's
		root_paths = {}
		for ID, value in build_conf.BLENDER_BUILDS.iteritems():
			root = value['root']
			root_paths.setdefault(root, []).append(ID)
 
		action = False
		# huston we have an update
 
		rebuild_ids = set()
 
		# add IDs from changed svn for update
		for ID_list in root_paths.itervalues():
			if build_up(ID_list[0]): # will be same for all builds that use this path
				rebuild_ids.update(ID_list)
 
		# add IDs with changes settings to update
		for ID in build_data:
			if ID not in build_data_prev or build_data[ID] != build_data_prev[ID]:
				rebuild_ids.add(ID)
 
		# build IDs we have tagged
		for ID in sorted(rebuild_ids):
			os.chdir( build_conf.BLENDER_BUILDS[ID]['root'] )
			build_do(ID)
			action = True
 
		build_data_prev = build_data
 
		if action:
			print ' ...sleeping, can cancel here with Ctrl+C'
 
		try:
			time.sleep(SLEEP)
		except KeyboardInterrupt:
			print 'Goodbye'
			return
 
		if action:
			print ' ...Attempting update now, hold tight'
		time.sleep(2) # some grace time.
 
 
if __name__ == '__main__':
	main()