Index: bin/autobuild ================================================================== --- bin/autobuild +++ bin/autobuild @@ -16,15 +16,24 @@ # 0. Resolve paths repository="$(cd "$(dirname "${repository}")" && echo "$(pwd)/$(basename "${repository}")")" || exit 1 workdir="$(mkdir -p "${workdir}" && cd "${workdir}" && pwd)" || exit 1 # 1. Register cleanup -tmpdir='' +tmpdirs=() function cleanup() { - if [ -n "${tmpdir}" ]; then - rm -rf "${tmpdir}" - fi + local tmpdir + + for tmpdir in "${tmpdirs[@]}"; do + case "${tmpdir}" in + /*) + if [ -n "${tmpdir}" -a -d "${tmpdir}" ]; then + rm -rf "${tmpdir}" + fi + ;; + esac + + done } trap cleanup EXIT # 2. Setup a sane environment ## 2.a. Abort on errors @@ -48,51 +57,166 @@ # 5. Get Fossil CI configuration from the trunk branch ## 5.a. Set default config ### 5.a.i. Set default variables excludedBranches=() -includedBranches=('') +includedBranches=() +excludedBuilders=() +includedBuilders=() buildCommands=('./autogen.sh || :' ./configure make) initCommands=() postCommands=() branchPostCommands=() testCommands=() +buildArtifacts=() builderID='' projectName="$(fossil info -R "${repository}" | awk '/^project-name:/{ sub(/^project-name: */, ""); gsub(/ /, ""); print; }')" tagSuffix='' tagPrefix='' +fossilUser='fossil-ci' +env=() # tagOmit='all|pass|redundant|none' tagOmit='none' # buildResultsUpload='fail|all|none' buildResultsUpload='fail' # testResultsUpload='fail|all|none' testResultsUpload='fail' +# postResultsUpload='fail|all|none' +postResultsUpload='all' ### 5.a.ii. Default functions function uploadBuildResults() { - ### XXX:TODO - : + local branch="$1" + local checkout="$2" + local buildlog="$3" + + uploadResultsLog "${branch}" "${checkout}" "${buildlog}" "build.txt" } function uploadTestResults() { - ### XXX:TODO - : + local branch="$1" + local checkout="$2" + local testlog="$3" + + uploadResultsLog "${branch}" "${checkout}" "${testlog}" "test.txt" +} + +function uploadPostResults() { + local branch="$1" + local checkout="$2" + local postlog="$3" + + uploadResultsLog "${branch}" "${checkout}" "${postlog}" "post.txt" +} + +function uploadResultsLog() { + local branch="$1" + local checkout="$2" + local log="$3" + local name="$4" + + local uvName="build/output/${branch}/${checkout}/logs/${name}" + + fossilUploadUV "${log}" "${uvName}" || return 1 + + echo "${uvName}" } function uploadBuildArtifacts() { - ### XXX:TODO - : + local branch="$1" + local checkout="$2" + local branchdir="$3" + shift; shift; shift + local artifacts=("$@") + + local name path + + for artifact in "${artifacts[@]}"; do + case "${artifact}" in + *=*) + path="$(echo "${artifact}" | cut -d '=' -f 1)" + name="$(echo "${artifact}" | cut -d '=' -f 2-)" + ;; + *) + path="${artifact}" + name="${artifact}" + ;; + esac + + if [ -f "${branchdir}/${path}" ]; then + fossilUploadUV "${branchdir}/${path}" "build/output/${branch}/${checkout}/artifacts/${name}" + fi + done } function fossilSetWikiPage() { - ### XXX:TODO - : + local page="$1" + local contents="$2" + + ( echo "${contents}" | fossil wiki create -R "${repository}" "${page}" --mimetype text/x-markdown 2>/dev/null >/dev/null ) || \ + ( echo "${contents}" | fossil wiki commit -R "${repository}" "${page}" --mimetype text/x-markdown 2>/dev/null >/dev/null ) || return 1 +} + +function fossilSetCheckinWikiPage() { + local checkin="$1" + local contents="$2" + + fossilSetWikiPage "checkin/${checkin}" "${contents}" } function fossilUploadUV() { - ### XXX:TODO - : + local file="$1" + local name="$2" + + fossil uv add "${file}" --as "${name}" -R "${repository}" +} + +function setupEnv() { + local envChange + local oldEnv + local envOverrideFile + + # Clear all environment variables except for a few safe ones + oldEnv="$(export -p | sed 's@^declare -x @@')" + unset $(echo "${oldEnv}" | cut -f 1 -d = | grep -Ev '^(TZ|SHELL|TMPDIR|HOME|PATH|USER|FOSSIL_CI_RUN_ID|FOSSIL_CI_CURRENT_BRANCH|FOSSIL_CI_CURRENT_PROJECT|FOSSIL_CI_CURRENT_REPOSITORY|FOSSIL_CI_CURRENT_BUILDER)$') + + envOverrideFile="${TMPDIR}/.fossil-ci-env-add-${FOSSIL_CI_RUN_ID}" + if [ -f "${envOverrideFile}" ]; then + local envOverride + + mapfile envOverride < "${envOverrideFile}" + env+=("${envOverride[@]}") + fi + + for envChange in "${env[@]}"; do + case "${envChange}" in + *=*) + export "${envChange}" + ;; + -*) + unset "${envChange:1}" + ;; + !HOME|!PATH|!USER|!TMPDIR|!TZ|!SHELL) + # Don't allow unsetting or restoring of critical environment variables + echo "Ignoring attempt to unset or restore critical environment variable: ${envChange}" >&2 + ;; + !*) + # Restore the old value of the environment variable, if it was set + eval "$(echo "${oldEnv}" | while IFS= read -r line; do + var="$(echo "${line}" | cut -f 1 -d =)" + if [ "${var}" = "${envChange:1}" ]; then + echo "export ${line}" + fi + done)" + ;; + *) + echo "Invalid environment variable: ${envChange}" >&2 + ;; + esac + done + + # XXX:TODO: Support secrets } ## 5.b. Read config config="$(fossil cat -R "${repository}" -r trunk "${fossilCIDir}/config" 2>/dev/null)" || : @@ -113,54 +237,75 @@ if [ -n "${builderID}" ]; then tagSuffix="-${builderID}" fi fi -# 6. Perform any configured initialization -for cmd in "${initCommands[@]}"; do - eval "${cmd}" >/dev/null 2>/dev/null || : -done - -# 7. For each branch, attempt to build -for branch in "${branches[@]}"; do - ## 7.a. Only process specific branches - ignoreBranch='1' - for testBranch in "${includedBranches[@]}"; do - if echo "${branch}" | grep "${testBranch}" >/dev/null; then - ignoreBranch='0' - - break - fi - done - - for testBranch in "${excludedBranches[@]}"; do - if echo "${branch}" | grep "${testBranch}" >/dev/null; then - ignoreBranch='1' - - break - fi - done - - if [ "${ignoreBranch}" = '1' ]; then - continue - fi +# 6. Create non-overridable commands +function addEnv() ( + set +x + local var="$1" + local value="$2" + + echo "${var}=${value@Q}" >> "${TMPDIR}/.fossil-ci-env-add-${FOSSIL_CI_RUN_ID}" +) + +function setupHome() { + local runTmpDir + + export TZ='UTC' + export SHELL='/bin/bash' + export TMPDIR="${workdir}/tmp" + + runTmpDir="$(mktemp -d)" + tmpdirs+=("${runTmpDir}") + + # Setup Home and TMPDIR to be within the workdir, so that any files created there will be automatically cleaned up when the workdir is cleaned up + export HOME="${runTmpDir}/home" + export TMPDIR="${runTmpDir}/tmp" + mkdir -p "${TMPDIR}" "${HOME}" +} + + +function runBranch() ( + set -e -o pipefail + + trap cleanup EXIT + + branch="$1" + + export FOSSIL_CI_CURRENT_BRANCH="${branch}" + export FOSSIL_CI_CURRENT_PROJECT="${projectName}" + export FOSSIL_CI_CURRENT_REPOSITORY="${repository}" + export FOSSIL_CI_CURRENT_BUILDER="${builderID}" + export FOSSIL_CI_RUN_ID="$(uuidgen -r)" + ## 7.b. Find the branch working directory branchdir="$(echo "${branch}" | sed 's@[^A-Za-z0-9-]@_@g')" + branchdir="${branchdir}-$(echo "${branch}" | openssl dgst -sha256 | sed 's@^.*= *@@' | cut -c 1-16)" builddir="${workdir}/build-output/branch-${branchdir}" branchdir="${workdir}/working-copies/branch-${branchdir}" mkdir -p "${branchdir}" ## 7.c. Determine current checkout ID - checkout="$(fossil info -R "${repository}" "${branch}" | awk '/^uuid:/{ print $3 "T" $4 "-" $2 }')" + checkout="$(fossil info -R "${repository}" "${branch}" | awk '/^hash:/{ print $3 "T" $4 "-" $2 }')" + fullCheckout="$(fossil timeline -R "${repository}" --branch "${branch}" --format %H --type ci --limit 1 | head -n 1)" builddir="${builddir}/checkout-${checkout}" ## 7.d. Determine if build even needs to occur if [ -d "${builddir}" ]; then - continue + exit 0 fi + mkdir -p "${builddir}" + + ## 7.d. Create other directories as needed + mkdir -p "${workdir}/tmp" + + + ## 7.d Setup the environment for the branch + setupHome ## 7.e. Check-out or update the branch ( cd "${branchdir}" || exit 1 @@ -167,19 +312,24 @@ if [ ! -f '.fslckout' ]; then fossil open "${repository}" "${branch}" --nested else fossil update "${branch}" fi - ) > "${builddir}/update.log" 2>&1 || continue + ) > "${builddir}/update.log" 2>&1 || exit 0 ## 7.f. Build and log the results, if there are any build_pass='1' ( cd "${branchdir}" || exit 1 for cmd in "${buildCommands[@]}"; do - ( eval "set -x; ${cmd}" ) || exit 1 + ( + # Prepare the environment for each step + setupEnv || exit 1 + + eval "set -x; ${cmd}" + ) || exit 1 done ) > "${builddir}/build.log" 2>&1 || build_pass='0' ### 7.g. Test the branch tests_pass='-1' @@ -187,11 +337,16 @@ tests_pass='1' ( cd "${branchdir}" || exit 1 for cmd in "${testCommands[@]}"; do - ( eval "set -x; ${cmd}" ) || exit 1 + ( + # Prepare the environment for each step + setupEnv || exit 1 + + eval "set -x; ${cmd}" + ) || exit 1 done ) > "${builddir}/test.log" 2>&1 || tests_pass='0' fi ### 7.h. Tag the branch with @@ -223,13 +378,19 @@ tagsToAddOpts=() for tag in "${tagsToAdd[@]}"; do tagsToAddOpts=("${tagsToAddOpts[@]}" --tag "${tagPrefix}${tag}${tagSuffix}") done - fossil amend -R "${repository}" "${branch}" "${tagsToAddOpts[@]}" > "${builddir}/update.log" 2>&1 + if [ "${#tagsToAddOpts[@]}" -gt 0 ]; then + fossil amend -R "${repository}" "${branch}" --user-override "${fossilUser}" "${tagsToAddOpts[@]}" > "${builddir}/update.log" 2>&1 + fi ### 7.i. Upload the logs somewhere if requested + uploadedLogByPhase_build='' + uploadedLogByPhase_test='' + uploadedLogByPhase_post='' + uploadBuildResults='0' case "${buildResultsUpload}-${build_pass}" in all-0|all-1|fail-0|pass-1) uploadBuildResults='1' ;; @@ -240,20 +401,27 @@ all-0|all-1|fail-0|pass-1) uploadTestResults='1' ;; esac - if [ "${uploadBuildResults}" ]; then - uploadBuildResults "${branch}" "${checkout}" "${builddir}/build.log" || : - fi + uploadPostResults='0' + case "${postResultsUpload}-${build_pass}" in + all-0|all-1|fail-0|pass-1) + uploadPostResults='1' + ;; + esac if [ "${uploadBuildResults}" ]; then - uploadTestResults "${branch}" "${checkout}" "${builddir}/test.log" || : + uploadedLogByPhase_build="$(uploadBuildResults "${branch}" "${checkout}" "${builddir}/build.log" || :)" + fi + + if [ "${uploadTestResults}" ]; then + uploadedLogByPhase_test="$(uploadTestResults "${branch}" "${checkout}" "${builddir}/test.log" || :)" fi ### 7.j. Upload build artifacts somewhere if requested - uploadBuildArtifacts "${branch}" "${checkout}" "${branchdir}" || : + uploadBuildArtifacts "${branch}" "${checkout}" "${branchdir}" "${buildArtifacts[@]}" || : ### 7.k. Get a list of tags tags=( $(fossil tag -R "${repository}" list "${branch}") ) # 8. Perform post-build work (e.g., for release engineering) @@ -261,9 +429,138 @@ cd "${branchdir}" || exit 1 for cmd in "${branchPostCommands[@]}"; do ( eval "set -x; ${cmd}" ) || exit 1 done ) > "${builddir}/post.log" 2>&1 || : + + if [ "${uploadPostResults}" ]; then + if [ -s "${builddir}/post.log" ]; then + uploadedLogByPhase_post="$(uploadPostResults "${branch}" "${checkout}" "${builddir}/post.log" || :)" + fi + fi + + if true; then + uploadedLogs=() + if [ -n "${uploadedLogByPhase_build}" ]; then + uploadedLogs+=("${uploadedLogByPhase_build}") + fi + if [ -n "${uploadedLogByPhase_test}" ]; then + uploadedLogs+=("${uploadedLogByPhase_test}") + fi + if [ -n "${uploadedLogByPhase_post}" ]; then + uploadedLogs+=("${uploadedLogByPhase_post}") + fi + + page="$( + echo "# Build results for ${branch} [${fullCheckout}]" + echo '## Build status' + if [ "${build_pass}" = '1' ]; then + echo '- Build: PASS' + if [ -n "${tests_pass}" ]; then + if [ "${tests_pass}" = '1' ]; then + echo '- Tests: PASS' + else + echo '- Tests: FAIL' + fi + fi + else + echo '- Build: FAIL' + echo '- Tests: N/A' + fi + echo '' + + echo '## Logs' + if [ "${#uploadedLogs[@]}" -gt 0 ]; then + if [ -n "${uploadedLogByPhase_build}" ]; then + echo "- [Build](/uv/${uploadedLogByPhase_build})" + fi + if [ -n "${uploadedLogByPhase_test}" ]; then + echo "- [Test](/uv/${uploadedLogByPhase_test})" + fi + if [ -n "${uploadedLogByPhase_post}" ]; then + echo "- [Post-build](/uv/${uploadedLogByPhase_post})" + fi + else + echo '*(none)*' + fi + echo '' + + echo '## Artifacts' + if [ "${#buildArtifacts[@]}" -gt 0 ]; then + for artifact in "${buildArtifacts[@]}"; do + echo "- ${artifact} $(ls -l "${branchdir}/${artifact}" 2>/dev/null || echo '(not found)')" + done + else + echo '*(none)*' + fi + )" + fossilSetCheckinWikiPage "${fullCheckout}" "${page}" + fi +) + +# 6. Create our user and set the default user to that +if ! fossil user default -R "${repository}" "${fossilUser}" >/dev/null 2>/dev/null; then + fossil user new -R "${repository}" "${fossilUser}" "${fossilUser}" "Fossil CI" --no-password >/dev/null 2>/dev/null + fossil user default -R "${repository}" "${fossilUser}" || ( + echo "Failed to create default user ${fossilUser}" >&2 + exit 1 + ) || exit 1 +fi + +# 6. Perform any configured initialization +for cmd in "${initCommands[@]}"; do + eval "${cmd}" >/dev/null 2>/dev/null || : +done + +# 7. For each branch, attempt to build +for branch in "${branches[@]}"; do + ## 7.a. Skip builders that we don't want + ignoreBranch='0' + for testBuilder in "${includedBuilders[@]}"; do + if echo "${branch}" | grep "${testBuilder}" >/dev/null; then + ignoreBranch='0' + + break + fi + done + + for testBuilder in "${excludedBuilders[@]}"; do + if echo "${branch}" | grep "${testBuilder}" >/dev/null; then + ignoreBranch='1' + + break + fi + done + + if [ "${ignoreBranch}" = '1' ]; then + continue + fi + unset ignoreBranch testBuilder + + ## 7.a. Only process specific branches + ignoreBranch='0' + for testBranch in "${includedBranches[@]}"; do + if echo "${branch}" | grep "${testBranch}" >/dev/null; then + ignoreBranch='0' + + break + fi + done + + for testBranch in "${excludedBranches[@]}"; do + if echo "${branch}" | grep "${testBranch}" >/dev/null; then + ignoreBranch='1' + + break + fi + done + + if [ "${ignoreBranch}" = '1' ]; then + continue + fi + unset ignoreBranch testBranch + + runBranch "${branch}" done # 9. Clean up any branches that no longer exist ## XXX:TODO Index: bin/generate-css-snippet ================================================================== --- bin/generate-css-snippet +++ bin/generate-css-snippet @@ -12,21 +12,34 @@ status="$2" platform="$3" os="$(echo "${platform}" | cut -f 2 -d -)" - imageProvidingURL='https://rkeene.org/images/icons/dynamic-status/' + imageProvidingURL='https://rkeene.org/images/icons/dynamic-status' url="${imageProvidingURL}/?os=${os}&status=${status}&action=${action}" echo "${url}" } for action in build tests; do for status in pass fail; do + tag="${action}-${status}" +cat << _EOF_ +.timelineTable a[href*="/timeline?r=${tag}&"] { + background: url("$(infoToStatusImageURL "${action}" "${status}" 'linux')"); + width: 56px; + height: 20px; + background-size: 56px 20px; + background-repeat: no-repeat; + color: rgba(0,0,0,0); + display: inline-block; + text-indent: -65536px; +} +_EOF_ for platform in "${platforms[@]}"; do tag="${action}-${status}-${platform}" cat << _EOF_ -.timelineTableCell a[href*="/timeline?r=${tag}&"] { +.timelineTable a[href*="/timeline?r=${tag}&"] { background: url("$(infoToStatusImageURL "${action}" "${status}" "${platform}")"); width: 56px; height: 20px; background-size: 56px 20px; background-repeat: no-repeat;