#! /usr/bin/env bash
repository="$1"
workdir="$2"
fossilCIDir="$3"
if [ -z "${repository}" -o -z "${workdir}" ]; then
echo "Usage: autobuild <repository> <workdir> [<fossil-ci-dir>]" >&2
exit 1
fi
if [ -z "${fossilCIDir}" ]; then
fossilCIDir='.fossil-ci'
fi
# 0. Resolve paths
repository="$(cd "$(dirname "${repository}")" && echo "$(pwd)/$(basename "${repository}")")" || exit 1
workdir="$(mkdir -p "${workdir}" && cd "${workdir}" && pwd)" || exit 1
# 1. Register cleanup
tmpdirs=()
function cleanup() {
local tmpdir
for tmpdir in "${tmpdirs[@]}"; do
case "${tmpdir}" in
/*)
if [ -n "${tmpdir}" -a -d "${tmpdir}" ]; then
rm -rf "${tmpdir}"
fi
;;
esac
done
}
trap cleanup EXIT
# 2. Setup a sane environment
## 2.a. Abort on errors
set -e
## 2.b. Universal timezone
export TZ=UTC
## 2.c. Basic locale
unset $(locale | cut -f 1 -d =)
## 2.d. Sane umask
umask 022
# 3. Update the Fossil repository
fossil config -R "${repository}" pull all >/dev/null 2>/dev/null || :
fossil sync -R "${repository}" >/dev/null 2>/dev/null || :
# 4. Get a list of branches
branches=( $(fossil branch -R "${repository}" list) )
# 5. Get Fossil CI configuration from the trunk branch
## 5.a. Set default config
### 5.a.i. Set default variables
excludedBranches=()
includedBranches=()
excludedBuilders=()
includedBuilders=()
buildCommands=('./autogen.sh || :' ./configure make)
initCommands=()
postCommands=()
branchPostCommands=()
testCommands=()
buildArtifacts=()
builderID=''
projectName="$(fossil info -R "${repository}" | awk '/^project-name:/{ sub(/^project-name: */, ""); gsub(/ /, ""); print; }')"
tagSuffix=''
tagPrefix=''
fossilUser='fossil-ci'
env=()
# tagOmit='all|pass|redundant|none'
tagOmit='none'
# buildResultsUpload='fail|all|none'
buildResultsUpload='fail'
# testResultsUpload='fail|all|none'
testResultsUpload='fail'
# postResultsUpload='fail|all|none'
postResultsUpload='all'
### 5.a.ii. Default functions
function uploadBuildResults() {
local branch="$1"
local checkout="$2"
local buildlog="$3"
uploadResultsLog "${branch}" "${checkout}" "${buildlog}" "build.txt"
}
function uploadTestResults() {
local branch="$1"
local checkout="$2"
local testlog="$3"
uploadResultsLog "${branch}" "${checkout}" "${testlog}" "test.txt"
}
function uploadPostResults() {
local branch="$1"
local checkout="$2"
local postlog="$3"
uploadResultsLog "${branch}" "${checkout}" "${postlog}" "post.txt"
}
function uploadResultsLog() {
local branch="$1"
local checkout="$2"
local log="$3"
local name="$4"
local uvName="build/output/${branch}/${checkout}/logs/${name}"
fossilUploadUV "${log}" "${uvName}" || return 1
echo "${uvName}"
}
function uploadBuildArtifacts() {
local branch="$1"
local checkout="$2"
local branchdir="$3"
shift; shift; shift
local artifacts=("$@")
local name path
for artifact in "${artifacts[@]}"; do
case "${artifact}" in
*=*)
path="$(echo "${artifact}" | cut -d '=' -f 1)"
name="$(echo "${artifact}" | cut -d '=' -f 2-)"
;;
*)
path="${artifact}"
name="${artifact}"
;;
esac
if [ -f "${branchdir}/${path}" ]; then
fossilUploadUV "${branchdir}/${path}" "build/output/${branch}/${checkout}/artifacts/${name}"
fi
done
}
function fossilSetWikiPage() {
local page="$1"
local contents="$2"
( echo "${contents}" | fossil wiki create -R "${repository}" "${page}" --mimetype text/x-markdown 2>/dev/null >/dev/null ) || \
( echo "${contents}" | fossil wiki commit -R "${repository}" "${page}" --mimetype text/x-markdown 2>/dev/null >/dev/null ) || return 1
}
function fossilSetCheckinWikiPage() {
local checkin="$1"
local contents="$2"
fossilSetWikiPage "checkin/${checkin}" "${contents}"
}
function fossilUploadUV() {
local file="$1"
local name="$2"
fossil uv add "${file}" --as "${name}" -R "${repository}"
}
function setupEnv() {
local envChange
local oldEnv
local envOverrideFile
# Clear all environment variables except for a few safe ones
oldEnv="$(export -p | sed 's@^declare -x @@')"
unset $(echo "${oldEnv}" | cut -f 1 -d = | grep -Ev '^(TZ|SHELL|TMPDIR|HOME|PATH|USER|FOSSIL_CI_RUN_ID|FOSSIL_CI_CURRENT_BRANCH|FOSSIL_CI_CURRENT_PROJECT|FOSSIL_CI_CURRENT_REPOSITORY|FOSSIL_CI_CURRENT_BUILDER)$')
envOverrideFile="${TMPDIR}/.fossil-ci-env-add-${FOSSIL_CI_RUN_ID}"
if [ -f "${envOverrideFile}" ]; then
local envOverride
mapfile envOverride < "${envOverrideFile}"
env+=("${envOverride[@]}")
fi
for envChange in "${env[@]}"; do
case "${envChange}" in
*=*)
export "${envChange}"
;;
-*)
unset "${envChange:1}"
;;
!HOME|!PATH|!USER|!TMPDIR|!TZ|!SHELL)
# Don't allow unsetting or restoring of critical environment variables
echo "Ignoring attempt to unset or restore critical environment variable: ${envChange}" >&2
;;
!*)
# Restore the old value of the environment variable, if it was set
eval "$(echo "${oldEnv}" | while IFS= read -r line; do
var="$(echo "${line}" | cut -f 1 -d =)"
if [ "${var}" = "${envChange:1}" ]; then
echo "export ${line}"
fi
done)"
;;
*)
echo "Invalid environment variable: ${envChange}" >&2
;;
esac
done
# XXX:TODO: Support secrets
}
## 5.b. Read config
config="$(fossil cat -R "${repository}" -r trunk "${fossilCIDir}/config" 2>/dev/null)" || :
## 5.c Load config
if [ -f ~/.fossil-ci/config ]; then
. ~/.fossil-ci/config
fi
eval "${config}"
if [ -f ~/.fossil-ci/"${projectName}"/config ]; then
. ~/.fossil-ci/"${projectName}"/config
fi
## 5.d. Post-process config
### 5.d.i. Add builderID as a tag suffix if none was given
if [ -z "${tagSuffix}" ]; then
if [ -n "${builderID}" ]; then
tagSuffix="-${builderID}"
fi
fi
# 6. Create non-overridable commands
function addEnv() (
set +x
local var="$1"
local value="$2"
echo "${var}=${value@Q}" >> "${TMPDIR}/.fossil-ci-env-add-${FOSSIL_CI_RUN_ID}"
)
function setupHome() {
local runTmpDir
export TZ='UTC'
export SHELL='/bin/bash'
export TMPDIR="${workdir}/tmp"
runTmpDir="$(mktemp -d)"
tmpdirs+=("${runTmpDir}")
# Setup Home and TMPDIR to be within the workdir, so that any files created there will be automatically cleaned up when the workdir is cleaned up
export HOME="${runTmpDir}/home"
export TMPDIR="${runTmpDir}/tmp"
mkdir -p "${TMPDIR}" "${HOME}"
}
function runBranch() (
set -e -o pipefail
trap cleanup EXIT
branch="$1"
export FOSSIL_CI_CURRENT_BRANCH="${branch}"
export FOSSIL_CI_CURRENT_PROJECT="${projectName}"
export FOSSIL_CI_CURRENT_REPOSITORY="${repository}"
export FOSSIL_CI_CURRENT_BUILDER="${builderID}"
export FOSSIL_CI_RUN_ID="$(uuidgen -r)"
## 7.b. Find the branch working directory
branchdir="$(echo "${branch}" | sed 's@[^A-Za-z0-9-]@_@g')"
branchdir="${branchdir}-$(echo "${branch}" | openssl dgst -sha256 | sed 's@^.*= *@@' | cut -c 1-16)"
builddir="${workdir}/build-output/branch-${branchdir}"
branchdir="${workdir}/working-copies/branch-${branchdir}"
mkdir -p "${branchdir}"
## 7.c. Determine current checkout ID
checkout="$(fossil info -R "${repository}" "${branch}" | awk '/^hash:/{ print $3 "T" $4 "-" $2 }')"
fullCheckout="$(fossil timeline -R "${repository}" --branch "${branch}" --format %H --type ci --limit 1 | head -n 1)"
builddir="${builddir}/checkout-${checkout}"
## 7.d. Determine if build even needs to occur
if [ -d "${builddir}" ]; then
exit 0
fi
mkdir -p "${builddir}"
## 7.d. Create other directories as needed
mkdir -p "${workdir}/tmp"
## 7.d Setup the environment for the branch
setupHome
## 7.e. Check-out or update the branch
(
cd "${branchdir}" || exit 1
if [ ! -f '.fslckout' ]; then
fossil open "${repository}" "${branch}" --nested
else
fossil update "${branch}"
fi
) > "${builddir}/update.log" 2>&1 || exit 0
## 7.f. Build and log the results, if there are any
build_pass='1'
(
cd "${branchdir}" || exit 1
for cmd in "${buildCommands[@]}"; do
(
# Prepare the environment for each step
setupEnv || exit 1
eval "set -x; ${cmd}"
) || exit 1
done
) > "${builddir}/build.log" 2>&1 || build_pass='0'
### 7.g. Test the branch
tests_pass='-1'
if [ "${build_pass}" = '1' ]; then
tests_pass='1'
(
cd "${branchdir}" || exit 1
for cmd in "${testCommands[@]}"; do
(
# Prepare the environment for each step
setupEnv || exit 1
eval "set -x; ${cmd}"
) || exit 1
done
) > "${builddir}/test.log" 2>&1 || tests_pass='0'
fi
### 7.h. Tag the branch with
tagsToAdd=()
if [ "${build_pass}" = '1' ]; then
case "${tagOmit}" in
all|pass|redundant)
;;
*)
tagsToAdd=("${tagsToAdd[@]}" build-pass)
;;
esac
if [ "${tests_pass}" = '1' ]; then
case "${tagOmit}" in
all|pass)
;;
*)
tagsToAdd=("${tagsToAdd[@]}" tests-pass)
;;
esac
else
tagsToAdd=("${tagsToAdd[@]}" tests-fail)
fi
else
tagsToAdd=("${tagsToAdd[@]}" build-fail)
fi
tagsToAddOpts=()
for tag in "${tagsToAdd[@]}"; do
tagsToAddOpts=("${tagsToAddOpts[@]}" --tag "${tagPrefix}${tag}${tagSuffix}")
done
if [ "${#tagsToAddOpts[@]}" -gt 0 ]; then
fossil amend -R "${repository}" "${branch}" --user-override "${fossilUser}" "${tagsToAddOpts[@]}" > "${builddir}/update.log" 2>&1
fi
### 7.i. Upload the logs somewhere if requested
uploadedLogByPhase_build=''
uploadedLogByPhase_test=''
uploadedLogByPhase_post=''
uploadBuildResults='0'
case "${buildResultsUpload}-${build_pass}" in
all-0|all-1|fail-0|pass-1)
uploadBuildResults='1'
;;
esac
uploadTestResults='0'
case "${buildResultsUpload}-${tests_pass}" in
all-0|all-1|fail-0|pass-1)
uploadTestResults='1'
;;
esac
uploadPostResults='0'
case "${postResultsUpload}-${build_pass}" in
all-0|all-1|fail-0|pass-1)
uploadPostResults='1'
;;
esac
if [ "${uploadBuildResults}" ]; then
uploadedLogByPhase_build="$(uploadBuildResults "${branch}" "${checkout}" "${builddir}/build.log" || :)"
fi
if [ "${uploadTestResults}" ]; then
uploadedLogByPhase_test="$(uploadTestResults "${branch}" "${checkout}" "${builddir}/test.log" || :)"
fi
### 7.j. Upload build artifacts somewhere if requested
uploadBuildArtifacts "${branch}" "${checkout}" "${branchdir}" "${buildArtifacts[@]}" || :
### 7.k. Get a list of tags
tags=( $(fossil tag -R "${repository}" list "${branch}") )
# 8. Perform post-build work (e.g., for release engineering)
(
cd "${branchdir}" || exit 1
for cmd in "${branchPostCommands[@]}"; do
( eval "set -x; ${cmd}" ) || exit 1
done
) > "${builddir}/post.log" 2>&1 || :
if [ "${uploadPostResults}" ]; then
if [ -s "${builddir}/post.log" ]; then
uploadedLogByPhase_post="$(uploadPostResults "${branch}" "${checkout}" "${builddir}/post.log" || :)"
fi
fi
if true; then
uploadedLogs=()
if [ -n "${uploadedLogByPhase_build}" ]; then
uploadedLogs+=("${uploadedLogByPhase_build}")
fi
if [ -n "${uploadedLogByPhase_test}" ]; then
uploadedLogs+=("${uploadedLogByPhase_test}")
fi
if [ -n "${uploadedLogByPhase_post}" ]; then
uploadedLogs+=("${uploadedLogByPhase_post}")
fi
page="$(
echo "# Build results for ${branch} [${fullCheckout}]"
echo '## Build status'
if [ "${build_pass}" = '1' ]; then
echo '- Build: PASS'
if [ -n "${tests_pass}" ]; then
if [ "${tests_pass}" = '1' ]; then
echo '- Tests: PASS'
else
echo '- Tests: FAIL'
fi
fi
else
echo '- Build: FAIL'
echo '- Tests: N/A'
fi
echo ''
echo '## Logs'
if [ "${#uploadedLogs[@]}" -gt 0 ]; then
if [ -n "${uploadedLogByPhase_build}" ]; then
echo "- [Build](/uv/${uploadedLogByPhase_build})"
fi
if [ -n "${uploadedLogByPhase_test}" ]; then
echo "- [Test](/uv/${uploadedLogByPhase_test})"
fi
if [ -n "${uploadedLogByPhase_post}" ]; then
echo "- [Post-build](/uv/${uploadedLogByPhase_post})"
fi
else
echo '*(none)*'
fi
echo ''
echo '## Artifacts'
if [ "${#buildArtifacts[@]}" -gt 0 ]; then
for artifact in "${buildArtifacts[@]}"; do
echo "- ${artifact} $(ls -l "${branchdir}/${artifact}" 2>/dev/null || echo '(not found)')"
done
else
echo '*(none)*'
fi
)"
fossilSetCheckinWikiPage "${fullCheckout}" "${page}"
fi
)
# 6. Create our user and set the default user to that
if ! fossil user default -R "${repository}" "${fossilUser}" >/dev/null 2>/dev/null; then
fossil user new -R "${repository}" "${fossilUser}" "${fossilUser}" "Fossil CI" --no-password >/dev/null 2>/dev/null
fossil user default -R "${repository}" "${fossilUser}" || (
echo "Failed to create default user ${fossilUser}" >&2
exit 1
) || exit 1
fi
# 6. Perform any configured initialization
for cmd in "${initCommands[@]}"; do
eval "${cmd}" >/dev/null 2>/dev/null || :
done
# 7. For each branch, attempt to build
for branch in "${branches[@]}"; do
## 7.a. Skip builders that we don't want
ignoreBranch='0'
for testBuilder in "${includedBuilders[@]}"; do
if echo "${branch}" | grep "${testBuilder}" >/dev/null; then
ignoreBranch='0'
break
fi
done
for testBuilder in "${excludedBuilders[@]}"; do
if echo "${branch}" | grep "${testBuilder}" >/dev/null; then
ignoreBranch='1'
break
fi
done
if [ "${ignoreBranch}" = '1' ]; then
continue
fi
unset ignoreBranch testBuilder
## 7.a. Only process specific branches
ignoreBranch='0'
for testBranch in "${includedBranches[@]}"; do
if echo "${branch}" | grep "${testBranch}" >/dev/null; then
ignoreBranch='0'
break
fi
done
for testBranch in "${excludedBranches[@]}"; do
if echo "${branch}" | grep "${testBranch}" >/dev/null; then
ignoreBranch='1'
break
fi
done
if [ "${ignoreBranch}" = '1' ]; then
continue
fi
unset ignoreBranch testBranch
runBranch "${branch}"
done
# 9. Clean up any branches that no longer exist
## XXX:TODO