From a4a222d62dcffaf93e3dd439c3f61836a4cd831e Mon Sep 17 00:00:00 2001 From: Rhody Lugo Date: Mon, 27 Nov 2017 09:39:05 -0400 Subject: [PATCH] use the same cache for all branches for appveyor --- .appveyor.yml | 9 +++-- .travis.yml | 14 +++---- SConstruct | 83 +++++++++++++++++++++++++++--------------- core/SCsub | 1 + drivers/SCsub | 1 + editor/SCsub | 1 + main/SCsub | 2 +- main/tests/SCsub | 2 +- methods.py | 7 +++- modules/SCsub | 1 + modules/freetype/SCsub | 1 + modules/gdnative/SCsub | 3 +- modules/recast/SCsub | 1 + modules/svg/SCsub | 2 + platform/SCsub | 4 +- platform/android/SCsub | 4 +- platform/iphone/SCsub | 1 + platform/osx/SCsub | 6 ++- platform/server/SCsub | 3 +- platform/uwp/SCsub | 1 + platform/windows/SCsub | 5 ++- platform/x11/SCsub | 6 ++- scene/SCsub | 1 + servers/SCsub | 1 + 24 files changed, 105 insertions(+), 55 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 7676362c1bed..aeee15e652ba 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -3,8 +3,8 @@ os: Visual Studio 2015 environment: HOME: "%HOMEDRIVE%%HOMEPATH%" PYTHON: C:\Python27 - SCONS_CACHE: "%HOME%\\scons_cache" - SCONS_CACHE_LIMIT: 128 + SCONS_CACHE_ROOT: "%HOME%\\scons_cache" + SCONS_CACHE_LIMIT: 512 matrix: - VS: C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat GD_PLATFORM: windows @@ -13,7 +13,7 @@ environment: ARCH: amd64 cache: - - "%SCONS_CACHE%" + - "%SCONS_CACHE_ROOT%" install: - SET "PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" @@ -25,6 +25,7 @@ before_build: - python --version - scons --version - cl.exe + - SET "SCONS_CACHE=%SCONS_CACHE_ROOT%\master" build_script: -- scons platform=%GD_PLATFORM% target=%TARGET% tools=%TOOLS% progress=no +- scons platform=%GD_PLATFORM% target=%TARGET% tools=%TOOLS% verbose=yes progress=no diff --git a/.travis.yml b/.travis.yml index 346ac82e9204..acab89e516cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ sudo: false env: global: - SCONS_CACHE=$HOME/.scons_cache - - SCONS_CACHE_LIMIT=128 + - SCONS_CACHE_LIMIT=1024 cache: directories: @@ -18,22 +18,22 @@ matrix: - env: STATIC_CHECKS=yes os: linux compiler: clang - - env: GODOT_TARGET=x11 TOOLS=yes + - env: GODOT_TARGET=x11 TOOLS=yes CACHE_NAME=${GODOT_TARGET}-gcc-tools os: linux compiler: gcc - - env: GODOT_TARGET=x11 TOOLS=no + - env: GODOT_TARGET=x11 TOOLS=no CACHE_NAME=${GODOT_TARGET}-clang os: linux compiler: clang - #- env: GODOT_TARGET=windows TOOLS=yes + #- env: GODOT_TARGET=windows TOOLS=yes CACHE_NAME=${GODOT_TARGET}-gcc-tools # os: linux # compiler: gcc - - env: GODOT_TARGET=android TOOLS=no + - env: GODOT_TARGET=android TOOLS=no CACHE_NAME=${GODOT_TARGET}-gcc os: linux compiler: gcc - - env: GODOT_TARGET=osx TOOLS=yes + - env: GODOT_TARGET=osx TOOLS=yes CACHE_NAME=${GODOT_TARGET}-clang-tools os: osx compiler: clang - #- env: GODOT_TARGET=iphone TOOLS=no + #- env: GODOT_TARGET=iphone TOOLS=no CACHE_NAME=${GODOT_TARGET}-clang # os: osx # compiler: clang diff --git a/SConstruct b/SConstruct index 6a32b1b51d13..7c171c8015f4 100644 --- a/SConstruct +++ b/SConstruct @@ -487,6 +487,7 @@ node_count_interval = 1 node_pruning = 8 # Number of nodes to process before prunning the cache if ('env' in locals()): node_count_fname = str(env.Dir('#')) + '/.scons_node_count' +show_progress = env['progress'] import time, math @@ -497,23 +498,26 @@ class cache_progress: self.path = path self.limit = limit self.exponent_scale = math.log(2) / half_life + if env['verbose'] and path != None: + screen.write('Current cache limit is ' + self.convert_size(limit) + ' (used: ' + self.convert_size(self.get_size(path)) + ')\n') self.pruning = node_pruning self.delete(self.file_list()) def __call__(self, node, *args, **kw): - global node_count, node_count_max, node_count_interval, node_count_fname, node_pruning - # Print the progress percentage - node_count += node_count_interval - if (node_count_max > 0 and node_count <= node_count_max): - screen.write('\r[%3d%%] ' % (node_count * 100 / node_count_max)) - screen.flush() - elif (node_count_max > 0 and node_count > node_count_max): - screen.write('\r[100%] ') - screen.flush() - else: - screen.write('\r[Initial build] ') - screen.flush() - # Prune if the number of nodes proccessed is 'node_pruning' or bigger + global node_count, node_count_max, node_count_interval, node_count_fname, node_pruning, show_progress + if show_progress: + # Print the progress percentage + node_count += node_count_interval + if (node_count_max > 0 and node_count <= node_count_max): + screen.write('\r[%3d%%] ' % (node_count * 100 / node_count_max)) + screen.flush() + elif (node_count_max > 0 and node_count > node_count_max): + screen.write('\r[100%] ') + screen.flush() + else: + screen.write('\r[Initial build] ') + screen.flush() + # Prune if the number of nodes processed is 'node_pruning' or bigger self.pruning -= node_count_interval if self.pruning <= 0: self.pruning = node_pruning @@ -522,8 +526,9 @@ class cache_progress: def delete(self, files): if len(files) == 0: return - # Utter something - screen.write('\rPurging %d %s from cache...\n' % (len(files), len(files) > 1 and 'files' or 'file')) + if env['verbose']: + # Utter something + screen.write('\rPurging %d %s from cache...\n' % (len(files), len(files) > 1 and 'files' or 'file')) map(os.remove, files) def file_list(self): @@ -557,22 +562,40 @@ class cache_progress: else: return [x[0] for x in file_stat[mark:]] + def convert_size(self, size_bytes): + if size_bytes == 0: + return "0 bytes" + size_name = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") + i = int(math.floor(math.log(size_bytes, 1024))) + p = math.pow(1024, i) + s = round(size_bytes / p, 2) + return "%s %s" % (int(s) if i == 0 else s, size_name[i]) + + def get_size(self, start_path = '.'): + total_size = 0 + for dirpath, dirnames, filenames in os.walk(start_path): + for f in filenames: + fp = os.path.join(dirpath, f) + total_size += os.path.getsize(fp) + return total_size + def progress_finish(target, source, env): - global node_count + global node_count, progressor with open(node_count_fname, 'w') as f: f.write('%d\n' % node_count) + progressor.delete(progressor.file_list()) -if 'env' in locals() and env['progress']: - try: - with open(node_count_fname) as f: - node_count_max = int(f.readline()) - except: - pass - cache_directory = os.environ.get("SCONS_CACHE") - # Simple cache pruning, attached to SCons' progress callback. Trim the - # cache directory to a size not larger than cache_limit. - cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", 1024)) * 1024 * 1024 - progress = cache_progress(cache_directory, cache_limit) - Progress(progress, interval = node_count_interval) - progress_finish_command = Command('progress_finish', [], progress_finish) - AlwaysBuild(progress_finish_command) +try: + with open(node_count_fname) as f: + node_count_max = int(f.readline()) +except: + pass +cache_directory = os.environ.get("SCONS_CACHE") +# Simple cache pruning, attached to SCons' progress callback. Trim the +# cache directory to a size not larger than cache_limit. +cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", 1024)) * 1024 * 1024 +progressor = cache_progress(cache_directory, cache_limit) +Progress(progressor, interval = node_count_interval) + +progress_finish_command = Command('progress_finish', [], progress_finish) +AlwaysBuild(progress_finish_command) diff --git a/core/SCsub b/core/SCsub index e9b21bc71b7d..be2034409ed3 100644 --- a/core/SCsub +++ b/core/SCsub @@ -106,5 +106,6 @@ SConscript('helper/SCsub') # Build it all as a library lib = env.Library("core", env.core_sources) +env.NoCache(lib) env.Prepend(LIBS=[lib]) Export('env') diff --git a/drivers/SCsub b/drivers/SCsub index 938927f3a9f0..d96d0ed7a98e 100644 --- a/drivers/SCsub +++ b/drivers/SCsub @@ -46,4 +46,5 @@ if env.split_drivers: else: env.add_source_files(env.drivers_sources, "*.cpp") lib = env.Library("drivers", env.drivers_sources) + env.NoCache(lib) env.Prepend(LIBS=[lib]) diff --git a/editor/SCsub b/editor/SCsub index 75ec422bd5c4..8a0e36b4a36f 100644 --- a/editor/SCsub +++ b/editor/SCsub @@ -475,6 +475,7 @@ if env['tools']: SConscript('plugins/SCsub') lib = env.Library("editor", env.editor_sources) + env.NoCache(lib) env.Prepend(LIBS=[lib]) Export('env') diff --git a/main/SCsub b/main/SCsub index ae63b94864a2..74a350ea0139 100644 --- a/main/SCsub +++ b/main/SCsub @@ -57,5 +57,5 @@ env.Command("#main/app_icon.gen.h", "#main/app_icon.png", make_app_icon) SConscript('tests/SCsub') lib = env.Library("main", env.main_sources) - +env.NoCache(lib) env.Prepend(LIBS=[lib]) diff --git a/main/tests/SCsub b/main/tests/SCsub index 03495c064913..bb44e9e5da39 100644 --- a/main/tests/SCsub +++ b/main/tests/SCsub @@ -10,5 +10,5 @@ Export('env') # SConscript('math/SCsub'); lib = env.Library("tests", env.tests_sources) - +env.NoCache(lib) env.Prepend(LIBS=[lib]) diff --git a/methods.py b/methods.py index 2be73f02d205..0bf5c01462bc 100644 --- a/methods.py +++ b/methods.py @@ -1496,6 +1496,7 @@ def split_lib(self, libname): if base != cur_base and len(list) > max_src: if num > 0: lib = env.Library(libname + str(num), list) + env.NoCache(lib) lib_list.append(lib) list = [] num = num + 1 @@ -1503,6 +1504,7 @@ def split_lib(self, libname): list.append(f) lib = env.Library(libname + str(num), list) + env.NoCache(lib) lib_list.append(lib) if len(lib_list) > 0: @@ -1510,11 +1512,14 @@ def split_lib(self, libname): if os.name == 'posix' and sys.platform == 'msys': env.Replace(ARFLAGS=['rcsT']) lib = env.Library(libname + "_collated", lib_list) + env.NoCache(lib) lib_list = [lib] lib_base = [] env.add_source_files(lib_base, "*.cpp") - lib_list.insert(0, env.Library(libname, lib_base)) + lib = env.Library(libname, lib_base) + env.NoCache(lib) + lib_list.insert(0, lib) env.Prepend(LIBS=lib_list) diff --git a/modules/SCsub b/modules/SCsub index c1cf5a6c1a21..ea8b58b8c5b2 100644 --- a/modules/SCsub +++ b/modules/SCsub @@ -18,5 +18,6 @@ for x in env.module_list: SConscript(x + "/SCsub") lib = env_modules.Library("modules", env.modules_sources) +env_modules.NoCache(lib) env.Prepend(LIBS=[lib]) diff --git a/modules/freetype/SCsub b/modules/freetype/SCsub index 19e384af7360..9169c7d67469 100644 --- a/modules/freetype/SCsub +++ b/modules/freetype/SCsub @@ -69,6 +69,7 @@ if env['builtin_freetype']: env.Append(CPPPATH=["#thirdparty/libpng"]) lib = env.Library("freetype_builtin", thirdparty_sources) + env.NoCache(lib) # Needs to be appended to arrive after libscene in the linker call, # but we don't want it to arrive *after* system libs, so manual hack # LIBS contains first SCons Library objects ("SCons.Node.FS.File object") diff --git a/modules/gdnative/SCsub b/modules/gdnative/SCsub index f5a593cf4ea0..1641d26cbf13 100644 --- a/modules/gdnative/SCsub +++ b/modules/gdnative/SCsub @@ -248,4 +248,5 @@ if ARGUMENTS.get('gdnative_wrapper', False): if not env.msvc: gd_wrapper_env.Append(CCFLAGS=['-fPIC']) - gd_wrapper_env.Library("#bin/gdnative_wrapper_code", [gensource]) + lib = gd_wrapper_env.Library("#bin/gdnative_wrapper_code", [gensource]) + gd_wrapper_env.NoCache(lib) diff --git a/modules/recast/SCsub b/modules/recast/SCsub index 500c0ec0556c..335ab6c16ed9 100644 --- a/modules/recast/SCsub +++ b/modules/recast/SCsub @@ -25,6 +25,7 @@ if env['builtin_recast']: env.Append(CPPPATH=[thirdparty_dir, thirdparty_dir + "/Include"]) lib = env.Library("recast_builtin", thirdparty_sources) + env.NoCache(lib) env.Append(LIBS=[lib]) # Godot source files diff --git a/modules/svg/SCsub b/modules/svg/SCsub index 5be936780825..1b71fbeca414 100644 --- a/modules/svg/SCsub +++ b/modules/svg/SCsub @@ -13,6 +13,8 @@ thirdparty_sources = [thirdparty_dir + file for file in thirdparty_sources] # env.add_source_files(env.modules_sources, thirdparty_sources) lib = env.Library("svg_builtin", thirdparty_sources) +env.NoCache(lib) + # Needs to be appended to arrive after libscene in the linker call, # but we don't want it to arrive *after* system libs, so manual hack # LIBS contains first SCons Library objects ("SCons.Node.FS.File object") diff --git a/platform/SCsub b/platform/SCsub index 4ef23ab053df..a362371f93b5 100644 --- a/platform/SCsub +++ b/platform/SCsub @@ -25,6 +25,8 @@ f.write(unreg_apis) f.close() platform_sources.append('register_platform_apis.gen.cpp') -env.Prepend(LIBS=env.Library('platform', platform_sources)) +lib = env.Library('platform', platform_sources) +env.NoCache(lib) +env.Prepend(LIBS=lib) Export('env') diff --git a/platform/android/SCsub b/platform/android/SCsub index 7fa026235934..74349cb0ad15 100644 --- a/platform/android/SCsub +++ b/platform/android/SCsub @@ -144,8 +144,8 @@ manifest = manifest.replace("$$ADD_APPATTRIBUTE_CHUNKS$$", env.android_appattrib pp_baseout.write(manifest) -env_android.SharedLibrary("#bin/libgodot", [android_objects], SHLIBSUFFIX=env["SHLIBSUFFIX"]) - +lib = env_android.SharedLibrary("#bin/libgodot", [android_objects], SHLIBSUFFIX=env["SHLIBSUFFIX"]) +env_android.NoCache(lib) lib_arch_dir = '' if env['android_arch'] == 'armv6': diff --git a/platform/iphone/SCsub b/platform/iphone/SCsub index 550dfdd7d606..5903934d7d93 100644 --- a/platform/iphone/SCsub +++ b/platform/iphone/SCsub @@ -18,6 +18,7 @@ iphone_lib = [ env_ios = env.Clone() ios_lib = env_ios.Library('iphone', iphone_lib) +env_ios.NoCache(ios_lib) def combine_libs(target=None, source=None, env=None): lib_path = target[0].srcnode().abspath diff --git a/platform/osx/SCsub b/platform/osx/SCsub index be3950bc6d1e..16223654ccbc 100644 --- a/platform/osx/SCsub +++ b/platform/osx/SCsub @@ -16,7 +16,9 @@ files = [ 'power_osx.cpp', ] -binary = env.Program('#bin/godot', files) +prog = env.Program('#bin/godot', files) +env.NoCache(prog) + if env["debug_symbols"] == "full" or env["debug_symbols"] == "yes": - env.AddPostAction(binary, make_debug) + env.AddPostAction(prog, make_debug) diff --git a/platform/server/SCsub b/platform/server/SCsub index 30195bb90820..9e7bfda1235a 100644 --- a/platform/server/SCsub +++ b/platform/server/SCsub @@ -7,4 +7,5 @@ common_server = [\ "os_server.cpp",\ ] -env.Program('#bin/godot_server', ['godot_server.cpp'] + common_server) +prog = env.Program('#bin/godot_server', ['godot_server.cpp'] + common_server) +env.NoCache(prog) diff --git a/platform/uwp/SCsub b/platform/uwp/SCsub index bbd329a7e569..ba375428a54d 100644 --- a/platform/uwp/SCsub +++ b/platform/uwp/SCsub @@ -20,6 +20,7 @@ if "build_angle" in env and env["build_angle"]: cmd = env.AlwaysBuild(env.ANGLE('libANGLE.lib', None)) prog = env.Program('#bin/godot', files) +env.NoCache(prog) if "build_angle" in env and env["build_angle"]: env.Depends(prog, [cmd]) diff --git a/platform/windows/SCsub b/platform/windows/SCsub index aa9eb3e69bca..2fdb801c2e5f 100644 --- a/platform/windows/SCsub +++ b/platform/windows/SCsub @@ -28,7 +28,8 @@ obj = env.RES(restarget, 'godot_res.rc') common_win.append(obj) -binary = env.Program('#bin/godot', ['godot_win.cpp'] + common_win, PROGSUFFIX=env["PROGSUFFIX"]) +prog = env.Program('#bin/godot', ['godot_win.cpp'] + common_win, PROGSUFFIX=env["PROGSUFFIX"]) +env.NoCache(prog) # Microsoft Visual Studio Project Generation if env['vsproj']: @@ -38,4 +39,4 @@ if env['vsproj']: if not os.getenv("VCINSTALLDIR"): if env["debug_symbols"] == "full" or env["debug_symbols"] == "yes": - env.AddPostAction(binary, make_debug_mingw) + env.AddPostAction(prog, make_debug_mingw) diff --git a/platform/x11/SCsub b/platform/x11/SCsub index aabc49149f1b..1eeee8380f2c 100644 --- a/platform/x11/SCsub +++ b/platform/x11/SCsub @@ -17,6 +17,8 @@ common_x11 = [ "power_x11.cpp", ] -binary = env.Program('#bin/godot', ['godot_x11.cpp'] + common_x11) +prog = env.Program('#bin/godot', ['godot_x11.cpp'] + common_x11) +env.NoCache(prog) + if env["debug_symbols"] == "full" or env["debug_symbols"] == "yes": - env.AddPostAction(binary, make_debug) + env.AddPostAction(prog, make_debug) diff --git a/scene/SCsub b/scene/SCsub index 513adeffda97..bec1b40ed627 100644 --- a/scene/SCsub +++ b/scene/SCsub @@ -31,6 +31,7 @@ SConscript('resources/SCsub') # Build it all as a library lib = env.Library("scene", env.scene_sources) +env.NoCache(lib) env.Prepend(LIBS=[lib]) Export('env') diff --git a/servers/SCsub b/servers/SCsub index df55010a36de..ad43faf72ef4 100644 --- a/servers/SCsub +++ b/servers/SCsub @@ -14,5 +14,6 @@ SConscript('visual/SCsub') SConscript('audio/SCsub') lib = env.Library("servers", env.servers_sources) +env.NoCache(lib) env.Prepend(LIBS=[lib])