<!-- Managed by Jenkins Job Builder --> TCWG CCC Update jenkins containers false false GLOBAL DISABLED 30 100 -1 -1 false false false false nodes Machines to run on tcwg && !tcwg-bmk-hw true allCases distro Distro version to use. default false force Whether to force update even with no changes in image false verbose Whether to be verbose true scripts_branch Scripts revision to use master false 1 1 0 true project false 2 origin +refs/heads/*:refs/remotes/origin/* +refs/changes/*:refs/changes/* https://gitlab.com/LinaroLtd/tcwg/jenkins-scripts.git $scripts_branch false jenkins-scripts false false false /home/tcwg-buildslave/snapshots-ref/jenkins-scripts.git 1 false tcwg-coordinator false false false false @daily true workspace/tcwg-update-jenkins-containers_$EXECUTOR_NUMBER/$NODE_NAME #!/bin/bash set -ex # Check if we need to update the image -- run with --dryrun true. ./jenkins-scripts/tcwg-update-host-containers.sh \ --distro "$distro" \ --jenkins_server "$JENKINS_URL" \ --dryrun true \ --node "$NODE_NAME" \ --force "$force" \ --verbose "$verbose" & res=0 && wait $! || res=$? if [ $res = 0 ]; then # Fast-path exit to avoid bringing the node offline. echo "$NODE_NAME is up-to-date" # Skip the rest and mark the build UNSTABLE (aka skipped). exit 125 elif [ $res = 125 ]; then echo "$NODE_NAME needs container update" elif [ $res != 0 ]; then echo "ERROR: container check failed" exit $res fi # We are about to update the container that is running this. # The plan is: # 1. Prevent new builds from starting by putting the node into # offline mode. # 2. Wait for current builds to finish. We detect this by # checking for children process of the jenkins client. # 3. Trigger a job on the master node to bring this node back # online. Without this we would restart the jenkins container, # but the node would still be marked as "offline" and no new # builds will be scheduled to it. # 4. Restart the container. # Mark the node offline. ssh -p$JENKINS_SSH_PORT -l $JENKINS_SSH_USER $JENKINS_SSH_SERVER \ offline-node "$NODE_NAME" \ -m "Updating_jenkins_container:$BUILD_URL" start_date=$(date +%s) rm -f timeout # Wait for current builds to finish. while true; do n_busy=$(source jenkins-scripts/jenkins-helpers.sh print_number_of_busy_executors "$NODE_NAME") if [ "$n_busy" = "1" ]; then # We are the only build left. break fi elapsed=$(($(date +%s) - $start_date)) elapsed=$(($elapsed / 60)) if [ "$elapsed" -gt "270" ]; then # We'll timeout in 30 minutes; give up on the update and # bring the node back online. # We budget 30 minutes for tcwg-cleanup-stale-workspaces.sh # below. touch timeout break fi # Wait for other builds to complete. sleep 60 done 125 SUCCESS 0 BLUE true SUCCESS 0 BLUE true node=$NODE_NAME build_num=$BUILD_NUMBER false tcwg-update-jenkins-containers-online-node ALWAYS false false false #!/bin/bash set -ex if [ -f timeout ]; then exit 125 fi # Cleanup workspace directory while the node is idle. ( set +e $WORKSPACE/jenkins-scripts/tcwg-cleanup-stale-workspaces.sh \ --days 3 --workspace_top $HOME/workspace ) ./jenkins-scripts/tcwg-update-host-containers.sh \ --distro "$distro" \ --dryrun false \ --node "$NODE_NAME" \ --force "$force" \ --verbose "$verbose" || exit 125 125 e0958a95-204f-4c14-a66c-5e2be6c5d50a false true true 300