From BlenderWiki

Jump to: navigation, search
Note: This is an archived version of the Blender Developer Wiki. The current and active wiki is available on wiki.blender.org.

BuildBot Script

Automatically build and upload on svn changes, can deal with multiple branches and build options.


build_conf.py

config file for the build bot, directories to build and arguments, any changes or additional builds will be detected and built/rebuild while build_bot.py runs

BLENDER_BUILDS = {}
 
trunk = '/home/ideasman42/build/blender'
 
BLENDER_BUILDS['linux_x86-64_lite'] = {\
	'root':trunk, \
	'args':'blenderlite BF_QUIET=1 BF_FANCY=0 WITH_BF_STATICCXX=1 BF_CXX_LIB_STATIC="/usr/lib/gcc/x86_64-linux-gnu/4.3/libstdc++.a" PLATFORM_LINKFLAGS="-pthread -static-libgcc"', \
	'strip':True, \
}
 
BLENDER_BUILDS['linux_x86-64_lite_debug'] = {\
	'root':trunk, \
	'args':BLENDER_BUILDS['linux_x86-64_lite']['args'] + ' BF_DEBUG=1 BF_DEBUG_FLAGS="-O0 -g3 -ggdb3 -fno-inline"', \
	'strip':False, \
}
 
BLENDER_BUILDS['win32_lite'] = {\
	'root':trunk, \
	'args':'blenderlite BF_CROSS=1 WITH_BF_STATICCXX=1  BF_CXX_LIB_STATIC="/home/ideasman42/build/lib/windows/pthreads/lib/libpthreadGC2.a /usr/lib/gcc/i586-mingw32msvc/4.2.1-sjlj/libstdc++.a /home/ideasman42/build/lib/windows/zlib/lib/libz.a /home/ideasman42/build/lib/windows/png/lib/libpng12.a  /home/ideasman42/build/lib/windows/jpeg/lib/libjpeg.a" BF_QUIET=0  BF_PTHREADS_LIB="" BF_PTHREADS_LIBPATH=""', \
	'strip':True, 'strip_cmd':'i586-mingw32msvc-strip',\
	'compress':'zip'
}
 
BLENDER_BUILDS['win32_lite_debug'] = {\
	'root':trunk, \
	'args':BLENDER_BUILDS['win32_lite']['args'] + ' BF_DEBUG=1 BF_DEBUG_FLAGS="-O0 -g3 -ggdb3 -fno-inline"', \
	'strip':False, \
	'compress':'zip'
}
build_bot.py
import os
import shutil # recursive delete
import time
SLEEP = 60
 
WPUT_BIN = '/opt/wput/bin/wput'
WPUT_TARGET = 'ftp://autobuilds@graphicall.org:xxxxxxxx@ftp.graphicall.org/'
WPUT_ARGS= '--dont-continue --reupload'
 
BLENDER_BIN = 'blender'
REBUILD_FULL = True # make clean every time?
VERBOSE = False
 
 
# Better to import so we can hardlink to a chroot
# from build_conf import BLENDER_BUILDS
 
import build_conf
 
def build_rev(ID):
	for l in os.popen('svn info %s' % build_conf.BLENDER_BUILDS[ID]['root']):
		if l.startswith('Last Changed Rev: '):
			return int(l.split()[-1])
	return -1
 
def build_up(ID):
	'''
	return True if we changed
	'''
 
	rev = build_rev(ID)
	if VERBOSE: print 'attempting SVN update, currently at rev:', rev
	os.popen('svn up %s' % build_conf.BLENDER_BUILDS[ID]['root']).read()
	if VERBOSE: print '...done'
	return rev != build_rev(ID)
 
 
 
 
def build_do(ID):
	txt_out = 'build_out_%s.txt' % ID
	txt_warn = 'build_warn_%s.txt' % ID
	dir_build = '../build/linux2_%s' % ID
	dir_install = '../blender-%s' % ID
	file_tar = '../blender_%s.tar' % ID
	file_zip = '../blender_%s.zip' % ID
	file_bin = os.path.join(build_conf.BLENDER_BUILDS[ID]['root'], dir_install, BLENDER_BIN)
 
	# remove old install files
	print '\tblender:', ID, '-- removing all install files'
	if os.path.exists(dir_install):
		shutil.rmtree(dir_install) #os.rmdir( dir_install )
 
	# opposed to a clean build
	if REBUILD_FULL:
		print '\tblender:', ID, '-- clean build...'
		if os.path.exists(dir_build):
			shutil.rmtree(dir_build) #os.rmdir( dir_install )
 
	# Use BF_CONFIG ?
 
	args = build_conf.BLENDER_BUILDS[ID]['args'] + ' '
	args += 'BF_FANCY=0 '
	args += 'BF_BUILDDIR=%s ' % dir_build
	args += 'BF_INSTALLDIR=%s ' % dir_install
 
	# os.system('python scons/scons.py clean')
	print '\tblender:', ID, '-- building blender...'
	os.system('python scons/scons.py %s  1> %s 2> %s' % (args, txt_out, txt_warn) )
	print "\tscons args:", args
	# os.system('python scons/scons.py %s' % args ) # for testing 
 
	# Is this a windows binary?
	if os.path.exists( file_bin + '.exe' ):
		file_bin = file_bin + '.exe'
 
	if not os.path.exists( file_bin ):
		print 'Build Failed! - Not Found:', file_bin
		return
 
	shutil.copyfile(txt_out, os.path.join( dir_install, os.path.basename(txt_out) ))
	shutil.copyfile(txt_warn, os.path.join( dir_install, os.path.basename(txt_warn) ))
 
	# strip the binary for smaller size?
	if build_conf.BLENDER_BUILDS[ID]['strip']:
		if 'strip_cmd' in build_conf.BLENDER_BUILDS[ID]:
			strip_cmd = build_conf.BLENDER_BUILDS[ID]['strip_cmd']
		else:
			strip_cmd = 'strip'
 
		os.system('%s -s "%s"' % (strip_cmd, file_bin))
 
	# Build a list of files to upload
	uploads = []
	'''
	if file_bin.endswith('.exe'): # hack for win32 blenderlite
		os.system('/opt/upx/upx --lzma -9 %s' % file_bin)
		file_bin_new = os.path.basename(file_bin)
		file_bin_new = file_bin_new.replace('blender.exe', 'blender_' + ID + '.exe')
		shutil.move(file_bin, file_bin_new)
		uploads.append(file_bin_new)
	else:
	'''
 
	if 1:
		if 'compress' in build_conf.BLENDER_BUILDS[ID] and build_conf.BLENDER_BUILDS[ID]['compress']=='zip':
			if os.path.exists(file_zip):
				os.remove(file_zip)
 
			cwd = os.getcwd()
			os.chdir(dir_install)
			os.system('zip -9 -r %s *' % (os.path.join(cwd, file_zip)))
			os.chdir(cwd)
			uploads.append(file_zip)
 
		else:
			print '\tblender:', ID, '-- tar files...'
			os.system('tar -cf %s %s' % (file_tar, dir_install))
			print '\tblender:', ID, '-- bzip2 tar files...'
 
			if os.path.exists('%s.bz2' % file_tar):
				os.remove('%s.bz2' % file_tar)
 
			os.system('bzip2 -9 %s' % file_tar)
 
			# TODO - wput
			print '\tblender:', ID, 'uploading'
			print '%s %s.bz2 %s' % (WPUT_BIN, file_tar, WPUT_TARGET)
 
			uploads.append(file_tar + '.bz2')
 
	uploads.append(txt_out)
	uploads.append(txt_warn)
 
	for f in uploads:
		os.system('%s %s %s %s' % (WPUT_BIN, f, WPUT_TARGET, WPUT_ARGS))
 
	print 'done...'
 
 
def build_data_str():
	'''
	Use this to see if build settings change
	'''
	build_data = {}
	for ID, value in build_conf.BLENDER_BUILDS.iteritems():
		build_data[ID] = str(value)
 
	return build_data
 
def main():
	# main build loop
 
	# Allow new builds to be added while the script runs
 
	build_data_prev = build_data_str()
 
	ok = True
	print "Starting build loop for", len(build_conf.BLENDER_BUILDS), 'blenders'
	while ok:
 
		# refresh settings
		reload(build_conf)
		build_data = build_data_str()
 
		# Get unique roots and link with ID's
		root_paths = {}
		for ID, value in build_conf.BLENDER_BUILDS.iteritems():
			root = value['root']
			root_paths.setdefault(root, []).append(ID)
 
		action = False
		# huston we have an update
 
		rebuild_ids = set()
 
		# add IDs from changed svn for update
		for ID_list in root_paths.itervalues():
			if build_up(ID_list[0]): # will be same for all builds that use this path
				rebuild_ids.update(ID_list)
 
		# add IDs with changes settings to update
		for ID in build_data:
			if ID not in build_data_prev or build_data[ID] != build_data_prev[ID]:
				rebuild_ids.add(ID)
 
		# build IDs we have tagged
		for ID in sorted(rebuild_ids):
			os.chdir( build_conf.BLENDER_BUILDS[ID]['root'] )
			build_do(ID)
			action = True
 
		build_data_prev = build_data
 
		if action:
			print ' ...sleeping, can cancel here with Ctrl+C'
 
		try:
			time.sleep(SLEEP)
		except KeyboardInterrupt:
			print 'Goodbye'
			return
 
		if action:
			print ' ...Attempting update now, hold tight'
		time.sleep(2) # some grace time.
 
 
if __name__ == '__main__':
	main()