From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <gentoo-commits+bounces-1200586-garchives=archives.gentoo.org@lists.gentoo.org>
Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80])
(using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits))
(No client certificate requested)
by finch.gentoo.org (Postfix) with ESMTPS id 2079B138359
for <garchives@archives.gentoo.org>; Sat, 29 Aug 2020 12:34:41 +0000 (UTC)
Received: from pigeon.gentoo.org (localhost [127.0.0.1])
by pigeon.gentoo.org (Postfix) with SMTP id 46C3CE0918;
Sat, 29 Aug 2020 12:34:40 +0000 (UTC)
Received: from smtp.gentoo.org (smtp.gentoo.org [IPv6:2001:470:ea4a:1:5054:ff:fec7:86e4])
(using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits))
(No client certificate requested)
by pigeon.gentoo.org (Postfix) with ESMTPS id 1EB5EE0918
for <gentoo-commits@lists.gentoo.org>; Sat, 29 Aug 2020 12:34:40 +0000 (UTC)
Received: from oystercatcher.gentoo.org (unknown [IPv6:2a01:4f8:202:4333:225:90ff:fed9:fc84])
(using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits))
(No client certificate requested)
by smtp.gentoo.org (Postfix) with ESMTPS id 1BF5E340C50
for <gentoo-commits@lists.gentoo.org>; Sat, 29 Aug 2020 12:34:39 +0000 (UTC)
Received: from localhost.localdomain (localhost [IPv6:::1])
by oystercatcher.gentoo.org (Postfix) with ESMTP id C284B2EE
for <gentoo-commits@lists.gentoo.org>; Sat, 29 Aug 2020 12:34:37 +0000 (UTC)
From: "Andreas K. Hüttel" <dilfridge@gentoo.org>
To: gentoo-commits@lists.gentoo.org
Content-Transfer-Encoding: 8bit
Content-type: text/plain; charset=UTF-8
Reply-To: gentoo-dev@lists.gentoo.org, "Andreas K. Hüttel" <dilfridge@gentoo.org>
Message-ID: <1598704417.e2d27f67f5c832e30b4683917e2bd7e28f22cfb9.dilfridge@gentoo>
Subject: [gentoo-commits] proj/releng:master commit in: tools/
X-VCS-Repository: proj/releng
X-VCS-Files: tools/catalyst-auto
X-VCS-Directories: tools/
X-VCS-Committer: dilfridge
X-VCS-Committer-Name: Andreas K. Hüttel
X-VCS-Revision: e2d27f67f5c832e30b4683917e2bd7e28f22cfb9
X-VCS-Branch: master
Date: Sat, 29 Aug 2020 12:34:37 +0000 (UTC)
Precedence: bulk
List-Post: <mailto:gentoo-commits@lists.gentoo.org>
List-Help: <mailto:gentoo-commits+help@lists.gentoo.org>
List-Unsubscribe: <mailto:gentoo-commits+unsubscribe@lists.gentoo.org>
List-Subscribe: <mailto:gentoo-commits+subscribe@lists.gentoo.org>
List-Id: Gentoo Linux mail <gentoo-commits.gentoo.org>
X-BeenThere: gentoo-commits@lists.gentoo.org
X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply
X-Archives-Salt: 55bae78d-e654-4490-aeaf-719c0e50eced
X-Archives-Hash: b1cc8a1abe950ffa0483ae3a74a7b5e0
commit: e2d27f67f5c832e30b4683917e2bd7e28f22cfb9
Author: Andreas K. Huettel <dilfridge <AT> gentoo <DOT> org>
AuthorDate: Sat Aug 22 19:11:51 2020 +0000
Commit: Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
CommitDate: Sat Aug 29 12:33:37 2020 +0000
URL: https://gitweb.gentoo.org/proj/releng.git/commit/?id=e2d27f67
catalyst-auto: Allow parallel building of spec sets
This assumes that the spec sets are logically independent from each other,
i.e., building different ABIs.
Code shamelessly adapted from locale-gen.
Signed-off-by: Andreas K. Huettel <dilfridge <AT> gentoo.org>
tools/catalyst-auto | 97 ++++++++++++++++++++++++++++++++++++-----------------
1 file changed, 66 insertions(+), 31 deletions(-)
diff --git a/tools/catalyst-auto b/tools/catalyst-auto
index 34238c01..484eaffc 100755
--- a/tools/catalyst-auto
+++ b/tools/catalyst-auto
@@ -23,6 +23,7 @@ testing=0
preclean=0
lastrun=0
lock_file=
+parallel_sets=1
usage() {
local msg=$1
@@ -38,6 +39,7 @@ Usage:
Options:
-c|--config Specifies the config file to use (required)
-C|--preclean Clean up loose artifacts from previous runs
+ -j|--jobs <n> Build <n> spec sets in parallel
-v|--verbose Send output of commands to console as well as log
-k|--keep-tmpdir Don't remove temp dir when build finishes
-t|--test Stop after mangling specs and copying files
@@ -133,6 +135,10 @@ parse_args() {
config_files+=("$1")
shift
;;
+ -j|--jobs)
+ parallel_sets="$1"
+ shift
+ ;;
-v|--verbose)
verbose=$(($verbose+1))
;;
@@ -385,47 +391,76 @@ run_catalyst_commands() {
timeprefix=()
which time >/dev/null && timeprefix=( "time" )
+ JOB_PIDS=()
+ JOB_RETS=()
+ JOB_IDX_S=0
+ JOB_IDX_E=0
+
for a in "" ${SETS}; do
- if [[ -z ${a} ]]; then
- specs_var="SPECS"
- optional_specs_var="OPTIONAL_SPECS"
- else
- specs_var="SET_${a}_SPECS"
- optional_specs_var="SET_${a}_OPTIONAL_SPECS"
+
+ if [[ $(( JOB_IDX_E - JOB_IDX_S )) == ${parallel_sets} ]] ; then
+ wait ${JOB_PIDS[$(( JOB_IDX_S++ ))]}
+ JOB_RETS+=( $? )
fi
- for i in ${!specs_var}; do
- LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log"
- specpath=$(readlink -f "${i}")
- run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c "${CATALYST_CONFIG}" -f "${specpath}"
- if [[ $? != 0 ]]; then
- build_failure=1
- send_email "Catalyst fatal build error - ${i}" "" "${LOGFILE}"
- continue 2
- else
- trigger_post_build "${a}" "${i}"
- fi
- done
+ (
- for i in ${!optional_specs_var}; do
- LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log"
- specpath=$(readlink -f "${i}")
- run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c "${CATALYST_CONFIG}" -f "${specpath}"
- if [[ $? != 0 ]]; then
- build_failure=1
- send_email "Catalyst non-fatal build error - ${i}" "" "${LOGFILE}"
- break
+ if [[ -z ${a} ]]; then
+ specs_var="SPECS"
+ optional_specs_var="OPTIONAL_SPECS"
else
- trigger_post_build "${a}" "${i}"
+ specs_var="SET_${a}_SPECS"
+ optional_specs_var="SET_${a}_OPTIONAL_SPECS"
fi
- done
- # Do not purge yet, because there might be interdendency between specs
- # in different build sets!
+ for i in ${!specs_var}; do
+ LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log"
+ specpath=$(readlink -f "${i}")
+ run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c "${CATALYST_CONFIG}" -f "${specpath}"
+ if [[ $? != 0 ]]; then
+ build_failure=1
+ send_email "Catalyst fatal build error - ${i}" "" "${LOGFILE}"
+ exit 1
+ else
+ trigger_post_build "${a}" "${i}"
+ fi
+ done
+
+ for i in ${!optional_specs_var}; do
+ LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:' -e 's:\.spec$::').log"
+ specpath=$(readlink -f "${i}")
+ run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c "${CATALYST_CONFIG}" -f "${specpath}"
+ if [[ $? != 0 ]]; then
+ build_failure=1
+ send_email "Catalyst non-fatal build error - ${i}" "" "${LOGFILE}"
+ break
+ else
+ trigger_post_build "${a}" "${i}"
+ fi
+ done
+
+ # Do not purge yet, because there might be interdendency between specs
+ # in different build sets!
+
+ update_symlinks
+
+ exit ${build_failure}
+
+ )&
+
+ JOB_PIDS+=( $! )
+ : $(( ++JOB_IDX_E ))
+
- update_symlinks
done
+ for (( i = JOB_IDX_S; i < JOB_IDX_E; ++i )) ; do
+ wait ${JOB_PIDS[i]}
+ JOB_RETS+=( $? )
+ done
+ build_failure=$(( 0 ${JOB_RETS[@]/#/+} ))
+
+
# Now do the cleanup
for a in "" ${SETS}; do
if [[ -z ${a} ]]; then