From 51cd819a2a316d95cb6fda9018fbe5774aae16e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krystian=20Bac=C5=82awski?= Date: Sun, 12 Nov 2017 19:15:13 +0100 Subject: [PATCH] Speed-up CircleCI builds for M68K and PowerPC. (#94) * Split out download action. * Use CircleCI cache to store downloads. * Run some make commands in parallel (-j option). --- .circleci/DATESTAMP | 1 + .circleci/config.yml | 20 ++++++++++++++++---- common.py | 7 +++++-- toolchain-m68k | 43 ++++++++++++++++++++++++------------------- toolchain-ppc | 35 +++++++++++++++++++---------------- 5 files changed, 65 insertions(+), 41 deletions(-) create mode 100644 .circleci/DATESTAMP diff --git a/.circleci/DATESTAMP b/.circleci/DATESTAMP new file mode 100644 index 0000000..1cd9650 --- /dev/null +++ b/.circleci/DATESTAMP @@ -0,0 +1 @@ +20171112-2 diff --git a/.circleci/config.yml b/.circleci/config.yml index 768bdea..2eec72b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,9 +10,15 @@ jobs: <<: *defaults steps: - checkout + - restore_cache: + keys: + - m68k-cache-{{ .Branch }}-{{ checksum ".circleci/DATESTAMP" }} + - run: './toolchain-m68k --quiet download' + - save_cache: + key: m68k-cache-{{ .Branch }}-{{ checksum ".circleci/DATESTAMP" }} + paths: + - .build-m68k - run: './toolchain-m68k --quiet build' - - store_artifacts: - path: m68k-amigaos - save_cache: key: m68k-amigaos-{{ .Branch }}-{{ .Revision }} paths: m68k-amigaos @@ -29,9 +35,15 @@ jobs: <<: *defaults steps: - checkout + - restore_cache: + keys: + - ppc-cache-{{ .Branch }}-{{ checksum ".circleci/DATESTAMP" }} + - run: './toolchain-ppc --quiet download' + - save_cache: + key: ppc-cache-{{ .Branch }}-{{ checksum ".circleci/DATESTAMP" }} + paths: + - .build-ppc - run: './toolchain-ppc --quiet build' - - store_artifacts: - path: ppc-amigaos - save_cache: key: ppc-amigaos-{{ .Branch }}-{{ .Revision }} paths: ppc-amigaos diff --git a/common.py b/common.py index 686c0f0..bf583e9 100644 --- a/common.py +++ b/common.py @@ -10,6 +10,7 @@ import contextlib from distutils import spawn, sysconfig import fileinput import os +from multiprocessing import cpu_count import shutil import site import subprocess @@ -417,7 +418,7 @@ def fetch(name, url): else: info('File "%s" already downloaded.', name) elif url.startswith('svn'): - execute('svn', 'export', '--quiet', url, name) + execute('svn', 'export', url, name) elif url.startswith('git'): if not path.exists(name): execute('git', 'clone', url, name) @@ -489,7 +490,7 @@ def configure(name, *confopts, **kwargs): @recipe('make', 2) -def make(name, target=None, makefile=None, **makevars): +def make(name, target=None, makefile=None, parallel=False, **makevars): info('running make "%s"', target) with cwd(path.join('{build}', name)): @@ -498,6 +499,8 @@ def make(name, target=None, makefile=None, **makevars): args = [target] + args if makefile is not None: args = ['-f', makefile] + args + if parallel: + args = ['-j%d' % cpu_count()] + args execute('make', *args) diff --git a/toolchain-m68k b/toolchain-m68k index c3dcb14..ed288b6 100755 --- a/toolchain-m68k +++ b/toolchain-m68k @@ -241,6 +241,19 @@ def touch_genfiles(dst): touch(h_file) +def download(): + with cwd('{archives}'): + for url in URLS: + if type(url) == tuple: + url, name = url[0], url[1] + else: + name = path.basename(url) + fetch(name, url) + + execute('git', 'submodule', 'init'); + execute('git', 'submodule', 'update'); + + def build(): for var in environ.keys(): if var not in ['_', 'LOGNAME', 'HOME', 'SHELL', 'TMPDIR', 'PWD']: @@ -310,28 +323,19 @@ def build(): unpack('python-lha', work_dir='{build}') pysetup('python-lha') - with cwd('{archives}'): - for url in URLS: - if type(url) == tuple: - url, name = url[0], url[1] - else: - name = path.basename(url) - fetch(name, url) - - execute('git', 'submodule', 'init'); - execute('git', 'submodule', 'update'); + download() unpack('{automake}') unpack('{m4}') configure('{m4}', '--prefix={host}') - make('{m4}') + make('{m4}', parallel=True) make('{m4}', 'install') unpack('{gawk}') update_autotools('{sources}/{gawk}') configure('{gawk}', '--prefix={host}') - make('{gawk}') + make('{gawk}', parallel=True) make('{gawk}', 'install') unpack('{flex}') @@ -342,19 +346,19 @@ def build(): unpack('{bison}') update_autotools('{sources}/{bison}/config') configure('{bison}', '--prefix={host}') - make('{bison}') + make('{bison}', parallel=True) make('{bison}', 'install') unpack('{texinfo}') update_autotools('{sources}/{texinfo}/build-aux') configure('{texinfo}', '--prefix={host}') - make('{texinfo}') + make('{texinfo}', parallel=True) make('{texinfo}', 'install') unpack('{autoconf}') update_autotools('{sources}/{autoconf}') configure('{autoconf}', '--prefix={host}') - make('{autoconf}') + make('{autoconf}', parallel=True) make('{autoconf}', 'install') prepare_target() @@ -418,7 +422,7 @@ def build(): '--target=m68k-amigaos', from_dir='{submodules}/{binutils}') touch_genfiles('{submodules}/{binutils}') - make('{binutils}') + make('{binutils}', parallel=True) make('{binutils}', 'install-binutils') make('{binutils}', 'install-gas') make('{binutils}', 'install-ld') @@ -442,6 +446,7 @@ def build(): touch_genfiles('{submodules}/{gcc}') touch('{submodules}/{gcc}/gcc/c-parse.gperf') touch('{submodules}/{gcc}/gcc/configure') + # parallel build fails for all-gcc make('{gcc}', 'all-gcc', MAKEINFO='makeinfo', CFLAGS_FOR_TARGET='-noixemul') make('{gcc}', 'install-gcc', @@ -493,11 +498,11 @@ def build(): make('{libdebug}', 'install') unpack('{clib2}', work_dir='{build}', top_dir='library') - make('{clib2}', makefile='GNUmakefile.68k') + make('{clib2}', makefile='GNUmakefile.68k', parallel=True) install_clib2() with env(CC=CC, CXX=CXX, CFLAGS=FLAGS, CXXFLAGS=FLAGS): - make('{gcc}', 'all-target', + make('{gcc}', 'all-target', parallel=True, MAKEINFO='makeinfo', CFLAGS_FOR_TARGET='-noixemul') make('{gcc}', 'install-target', MAKEINFO='makeinfo', CFLAGS_FOR_TARGET='-noixemul') @@ -719,7 +724,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description='Build cross toolchain.') parser.add_argument('action', choices=['build', 'list-sdk', 'install-sdk', 'clean', - 'test'], + 'test', 'download'], default='build', help='perform action') parser.add_argument('args', metavar='ARGS', type=str, nargs='*', help='action arguments') diff --git a/toolchain-ppc b/toolchain-ppc index 5779d71..e216025 100755 --- a/toolchain-ppc +++ b/toolchain-ppc @@ -48,6 +48,16 @@ def prepare_sdk(): unpack('SDK/newlib', top_dir='newlib', dst_dir=path.join(target, 'newlib')) +def download(): + with cwd('{archives}'): + for url in URLS: + if type(url) == tuple: + url, name = url[0], url[1] + else: + name = path.basename(url) + fetch(name, url) + + def build(): for var in environ.keys(): if var not in ['_', 'LOGNAME', 'HOME', 'SHELL', 'TMPDIR', 'PWD']: @@ -107,21 +117,14 @@ def build(): unpack('python-lha', work_dir='{build}') pysetup('python-lha') - with cwd('{archives}'): - for url in URLS: - if type(url) == tuple: - url, name = url[0], url[1] - else: - name = path.basename(url) - fetch(name, url) + download() unpack('{automake}') - unpack('{texinfo}') update_autotools('{sources}/{texinfo}/build-aux') configure('{texinfo}', '--prefix={host}') - make('{texinfo}') + make('{texinfo}', parallel=True) make('{texinfo}', 'install') unpack('{gmp}') @@ -129,7 +132,7 @@ def build(): configure('{gmp}', '--disable-shared', '--prefix={host}') - make('{gmp}') + make('{gmp}', parallel=True) make('{gmp}', 'install') unpack('{mpfr}') @@ -138,7 +141,7 @@ def build(): '--disable-shared', '--prefix={host}', '--with-gmp={host}') - make('{mpfr}') + make('{mpfr}', parallel=True) make('{mpfr}', 'install') unpack('{mpc}') @@ -148,7 +151,7 @@ def build(): '--prefix={host}', '--with-gmp={host}', '--with-mpfr={host}') - make('{mpc}') + make('{mpc}', parallel=True) make('{mpc}', 'install') unpack('{isl}') @@ -157,7 +160,7 @@ def build(): '--disable-shared', '--prefix={host}', '--with-gmp-prefix={host}') - make('{isl}') + make('{isl}', parallel=True) make('{isl}', 'install') unpack('{cloog}') @@ -168,7 +171,7 @@ def build(): '--with-isl=system', '--with-gmp-prefix={host}', '--with-isl-prefix={host}') - make('{cloog}') + make('{cloog}', parallel=True) make('{cloog}', 'install') with env(CFLAGS='-Wno-error'): @@ -177,7 +180,7 @@ def build(): '--prefix={prefix}', '--target={target}', from_dir='{archives}/{binutils}') - make('{binutils}') + make('{binutils}', parallel=True) make('{binutils}', 'install') prepare_sdk() @@ -231,7 +234,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description='Build cross toolchain.') parser.add_argument('action', - choices=['build', 'clean'], + choices=['build', 'clean', 'download'], default='build', help='perform action') parser.add_argument('args', metavar='ARGS', type=str, nargs='*', help='action arguments')