Use 2 stages for MacOS builds

native-api · native-api · commit 9d020bc045f8 · 2019-03-23T21:30:41.000+03:00
* Add a dedicated stage to each OSX build to download/build brew packages
* Remove bottle building logic and the corresponding trace output from the project build stage
diff --git a/.travis.yml b/.travis.yml
@@ -34,144 +34,232 @@ cache:
     # `cache: ccache: true` has no effect if `language:` is not `c` or `cpp`
     - $HOME/.ccache
 
+# Add more cache stages (s2 etc) and corresponding OSX jobs like s1
+# if brew builds start to take longer than one Travis time limit
+stages:
+  - s1
+  - final
+    
 matrix:
   fast_finish: true
   include:
 
     # default builds for MacOS
-    - os: osx
+    - &osx-10
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=2.7
         - ENABLE_CONTRIB=0
         - ENABLE_HEADLESS=0
-    - os: osx
+      stage: final
+    - <<: *osx-10
+      stage: s1
+    - &osx-30
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=3.4
         - ENABLE_CONTRIB=0
         - ENABLE_HEADLESS=0
-    - os: osx
+      stage: final
+    - <<: *osx-30
+      stage: s1
+    - &osx-40
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=3.5
         - ENABLE_CONTRIB=0
         - ENABLE_HEADLESS=0
-    - os: osx
+      stage: final
+    - <<: *osx-40
+      stage: s1
+    - &osx-50
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=3.6
         - ENABLE_CONTRIB=0
         - ENABLE_HEADLESS=0
-    - os: osx
+      stage: final
+    - <<: *osx-50
+      stage: s1
+    - &osx-60
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=3.7
         - ENABLE_CONTRIB=0
         - ENABLE_HEADLESS=0
         - TEST_DEPENDS=numpy==1.14.5
+      stage: final
+    - <<: *osx-60
+      stage: s1
 
     # headless builds for MacOS
-    - os: osx
+    - &osx-70
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=2.7
         - ENABLE_CONTRIB=0
         - ENABLE_HEADLESS=1
-    - os: osx
+      stage: final
+    - <<: *osx-70
+      stage: s1
+    - &osx-80
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=3.4
         - ENABLE_CONTRIB=0
         - ENABLE_HEADLESS=1
-    - os: osx
+      stage: final
+    - <<: *osx-80
+      stage: s1
+    - &osx-90
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=3.5
         - ENABLE_CONTRIB=0
         - ENABLE_HEADLESS=1
-    - os: osx
+      stage: final
+    - <<: *osx-90
+      stage: s1
+    - &osx-100
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=3.6
         - ENABLE_CONTRIB=0
         - ENABLE_HEADLESS=1
-    - os: osx
+      stage: final
+    - <<: *osx-100
+      stage: s1
+    - &osx-110
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=3.7
         - ENABLE_CONTRIB=0
         - ENABLE_HEADLESS=1
         - TEST_DEPENDS=numpy==1.14.5
+      stage: final
+    - <<: *osx-110
+      stage: s1
 
     # Contrib builds for MacOS
-    - os: osx
+    - &osx-120
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=2.7
         - ENABLE_CONTRIB=1
         - ENABLE_HEADLESS=0
-    - os: osx
+      stage: final
+    - <<: *osx-120
+      stage: s1
+    - &osx-130
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=3.4
         - ENABLE_CONTRIB=1
         - ENABLE_HEADLESS=0
-    - os: osx
+      stage: final
+    - <<: *osx-130
+      stage: s1
+    - &osx-140
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=3.5
         - ENABLE_CONTRIB=1
         - ENABLE_HEADLESS=0
-    - os: osx
+      stage: final
+    - <<: *osx-140
+      stage: s1
+    - &osx-150
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=3.6
         - ENABLE_CONTRIB=1
         - ENABLE_HEADLESS=0
-    - os: osx
+      stage: final
+    - <<: *osx-150
+      stage: s1
+    - &osx-160
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=3.7
         - ENABLE_CONTRIB=1
         - ENABLE_HEADLESS=0
         - TEST_DEPENDS=numpy==1.14.5
+      stage: final
+    - <<: *osx-160
+      stage: s1
 
     # headless contrib builds for MacOS
-    - os: osx
+    - &osx-170
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=2.7
         - ENABLE_CONTRIB=1
         - ENABLE_HEADLESS=1
-    - os: osx
+      stage: final
+    - <<: *osx-170
+      stage: s1
+    - &osx-180
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=3.4
         - ENABLE_CONTRIB=1
         - ENABLE_HEADLESS=1
-    - os: osx
+      stage: final
+    - <<: *osx-180
+      stage: s1
+    - &osx-190
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=3.5
         - ENABLE_CONTRIB=1
         - ENABLE_HEADLESS=1
-    - os: osx
+      stage: final
+    - <<: *osx-190
+      stage: s1
+    - &osx-200
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=3.6
         - ENABLE_CONTRIB=1
         - ENABLE_HEADLESS=1
-    - os: osx
+      stage: final
+    - <<: *osx-200
+      stage: s1
+    - &osx-210
+      os: osx
       osx_image: xcode8.3
       env:
         - MB_PYTHON_VERSION=3.7
         - ENABLE_CONTRIB=1
         - ENABLE_HEADLESS=1
         - TEST_DEPENDS=numpy==1.14.5
+      stage: final
+    - <<: *osx-210
+      stage: s1
 
     # default builds for Linux
     - os: linux
+      # the following jobs will use the same stage name by default
+      stage: s1
       env:
         - MB_PYTHON_VERSION=2.7
         - ENABLE_CONTRIB=0
diff --git a/config.sh b/config.sh
@@ -37,11 +37,6 @@ if [ -n "$IS_OSX" ]; then
         "ffmpeg_opencv 10" \
     )
 
-    #Contrib adds significantly to project's build time
-    if [ "$ENABLE_CONTRIB" -eq 1 ]; then
-        BREW_TIME_LIMIT=$((BREW_TIME_LIMIT - 10*60))
-    fi
-
     function generate_ffmpeg_formula {
         local FF="ffmpeg"
         local LFF="ffmpeg_opencv"
@@ -92,9 +87,14 @@ function pre_build {
 
   if [ -n "$IS_OSX" ]; then
     echo "Running for OSX"
+    
+    local CACHE_STAGE; (echo "$TRAVIS_BUILD_STAGE_NAME" | grep -qiF "final") || CACHE_STAGE=1
 
-    brew update
-    brew_add_local_bottles
+    #after the cache stage, all bottles and Homebrew metadata should be already cached locally
+    if [ -n "$CACHE_STAGE" ]; then
+        brew update
+        brew_add_local_bottles
+    fi
 
     # Don't query analytical info online on `brew info`,
     #  this takes several seconds and we don't need it
@@ -104,14 +104,25 @@ function pre_build {
     echo 'Installing QT4'
     brew tap | grep -qxF cartr/qt4 || brew tap cartr/qt4
     brew tap --list-pinned | grep -qxF cartr/qt4 || brew tap-pin cartr/qt4
-    brew_install_and_cache_within_time_limit qt@4 || { [ $? -gt 1 ] && return 2 || return 0; }
+    if [ -n "$CACHE_STAGE" ]; then
+        brew_install_and_cache_within_time_limit qt@4 || { [ $? -gt 1 ] && return 2 || return 0; }
+    else
+        brew install qt@4
+    fi
 
     echo 'Installing FFmpeg'
 
-    generate_ffmpeg_formula
-    local IS_BOTTLE_AVAILABLE; _brew_is_bottle_available ffmpeg_opencv && IS_BOTTLE_AVAILABLE=1 || IS_BOTTLE_AVAILABLE=0
-    _brew_install_and_cache ffmpeg_opencv "$IS_BOTTLE_AVAILABLE"
-    #brew_install_and_cache_within_time_limit ffmpeg_opencv || { [ $? -gt 1 ] && return 2 || return 0; }
+    if [ -n "$CACHE_STAGE" ]; then
+        generate_ffmpeg_formula
+        brew_install_and_cache_within_time_limit ffmpeg_opencv || { [ $? -gt 1 ] && return 2 || return 0; }
+    else
+        brew install ffmpeg_opencv
+    fi
+
+    if [ -n "$CACHE_STAGE" ]; then
+        brew_go_bootstrap_mode 0
+        return 0
+    fi
     
     # Have to install macpython late to avoid conflict with Homebrew Python update
     before_install
diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh
@@ -10,7 +10,8 @@ BREW_TIME_START=$(($TRAVIS_TIMER_START_TIME/10**9))
 
 # If after a package is built, elapsed time is more than this many seconds, fail the build but save Travis cache
 # The cutoff moment should leave enough time till Travis' job time limit to process the main project.
-BREW_TIME_LIMIT=$((30*60))
+#  Since we have moved deps into a separate stage, we don't need to leave time for the project any more
+BREW_TIME_LIMIT=$((38*60))
 # If a slow-building package is about to be built and the projected build end moment is beyond this many seconds,
 # skip that build, fail the Travis job and save Travis cache.
 # This cutoff should leave enough time for before_cache and cache save.
@@ -207,6 +208,7 @@ function brew_cache_cleanup {
 function brew_go_bootstrap_mode {
 # Can be overridden
 # Terminate the build but ensure saving the cache
+    local EXIT_CODE=${1:-1}
 
     echo "Going into cache bootstrap mode"
     
@@ -223,7 +225,7 @@ function brew_go_bootstrap_mode {
         
         # Travis runs user scripts via `eval` i.e. in the same shell process.
         # So have to unset errexit in order to get to cache save stage
-        set +e; return 1
+        set +e; return '"$EXIT_CODE"'
     }'    
 }
 
