Commit 7fa27d27 authored by Ivan Mazhukin's avatar Ivan Mazhukin

avoid remote sync race in parallel runs

parent 7b8ae14a
...@@ -26,6 +26,8 @@ LOG_ROOT="$DEFAULT_LOG_ROOT" ...@@ -26,6 +26,8 @@ LOG_ROOT="$DEFAULT_LOG_ROOT"
LOG_FILE="" LOG_FILE=""
REMOTE_ARGS=() REMOTE_ARGS=()
REMOTE_SYNC_DIR="" REMOTE_SYNC_DIR=""
PARALLEL_REMOTE_SOURCE_DIR=""
PARALLEL_REMOTE_SYNC_DIR=""
RUN_TOKEN="$(date +%Y%m%d-%H%M%S)-$$" RUN_TOKEN="$(date +%Y%m%d-%H%M%S)-$$"
SYSTEM_INPUTS=() SYSTEM_INPUTS=()
PRESET_NAMES=() PRESET_NAMES=()
...@@ -336,24 +338,27 @@ find_remote_builder_source_path() { ...@@ -336,24 +338,27 @@ find_remote_builder_source_path() {
remote_sync_dir_for_source() { remote_sync_dir_for_source() {
local source_dir="$1" local source_dir="$1"
local local_user base_name local local_user base_name suffix
local_user="$(id -un)" local_user="$(id -un)"
base_name="$(basename "$source_dir")" base_name="$(basename "$source_dir")"
printf '/tmp/epm-docker-test-sync/%s-%s-%s\n' \ suffix="$(sanitize_name "${3:-${SYSTEM_IMAGE:-run}}")"
printf '/tmp/epm-docker-test-sync/%s-%s-%s-%s\n' \
"$(sanitize_name "$local_user")" \ "$(sanitize_name "$local_user")" \
"$(sanitize_name "$base_name")" \ "$(sanitize_name "$base_name")" \
"$(sanitize_name "$RUN_TOKEN")" "$(sanitize_name "$RUN_TOKEN")" \
"$suffix"
} }
sync_local_source_to_remote() { sync_local_source_to_remote() {
local source_dir="$1" local source_dir="$1"
local target="$2" local target="$2"
local suffix="${3:-}"
local remote_dir local remote_dir
require_command rsync require_command rsync
remote_dir="$(remote_sync_dir_for_source "$source_dir")" remote_dir="$(remote_sync_dir_for_source "$source_dir" "$target" "$suffix")"
info "Remote eepm tree not found in /srv; syncing local tree to $target:$remote_dir" info "Remote eepm tree not found in /srv; syncing local tree to $target:$remote_dir"
...@@ -647,7 +652,9 @@ build_remote_args() { ...@@ -647,7 +652,9 @@ build_remote_args() {
target="$(remote_target)" target="$(remote_target)"
if [[ "$SOURCE_KIND" == "local" || "$SOURCE_KIND" == "explicit" ]]; then if [[ "$SOURCE_KIND" == "local" || "$SOURCE_KIND" == "explicit" ]]; then
if remote_source="$(find_remote_builder_source_path "$target")"; then if [[ -n "$PARALLEL_REMOTE_SOURCE_DIR" ]]; then
remote_source="$PARALLEL_REMOTE_SOURCE_DIR"
elif remote_source="$(find_remote_builder_source_path "$target")"; then
info "Remote eepm tree found at: $remote_source" info "Remote eepm tree found at: $remote_source"
else else
if [[ "$SOURCE_KIND" == "local" ]]; then if [[ "$SOURCE_KIND" == "local" ]]; then
...@@ -876,6 +883,66 @@ run_for_system_parallel() { ...@@ -876,6 +883,66 @@ run_for_system_parallel() {
return 0 return 0
} }
prepare_parallel_runner() {
case "$RUN_MODE" in
auto)
if can_use_local_docker; then
EFFECTIVE_RUN_MODE="local"
else
warn "Local Docker is unavailable; falling back to ssh runner"
EFFECTIVE_RUN_MODE="remote"
fi
;;
local|remote)
EFFECTIVE_RUN_MODE="$RUN_MODE"
;;
*)
fatal "Unsupported run mode: $RUN_MODE"
;;
esac
}
prepare_parallel_remote_source() {
local target
local local_source
local remote_source
PARALLEL_REMOTE_SOURCE_DIR=""
PARALLEL_REMOTE_SYNC_DIR=""
[[ "${EFFECTIVE_RUN_MODE:-$RUN_MODE}" == "remote" ]] || return 0
[[ "$SOURCE_KIND" == "local" || "$SOURCE_KIND" == "explicit" ]] || return 0
target="$(remote_target)"
if remote_source="$(find_remote_builder_source_path "$target")"; then
info "Remote eepm tree found at: $remote_source"
PARALLEL_REMOTE_SOURCE_DIR="$remote_source"
return 0
fi
if [[ "$SOURCE_KIND" == "local" ]]; then
local_source="$(resolve_checked_source_path "${SOURCE_PATH:-$(pwd -P)}")" || exit 1
else
local_source="$(resolve_checked_source_path "$SOURCE_PATH")" || exit 1
fi
remote_source="$(sync_local_source_to_remote "$local_source" "$target" parallel)" || exit 1
PARALLEL_REMOTE_SOURCE_DIR="$remote_source"
PARALLEL_REMOTE_SYNC_DIR="$remote_source"
}
cleanup_parallel_remote_source() {
local target
[[ -n "$PARALLEL_REMOTE_SYNC_DIR" ]] || return 0
target="$(remote_target)"
if ! cleanup_remote_sync_dir "$target" "$PARALLEL_REMOTE_SYNC_DIR"; then
warn "Failed to clean up remote sync dir: $PARALLEL_REMOTE_SYNC_DIR"
fi
PARALLEL_REMOTE_SYNC_DIR=""
PARALLEL_REMOTE_SOURCE_DIR=""
}
parse_args() { parse_args() {
local positional=() local positional=()
...@@ -1027,6 +1094,7 @@ main() { ...@@ -1027,6 +1094,7 @@ main() {
local result_dir local result_dir
local i sys res log local i sys res log
result_dir="$(mktemp -d "${TMPDIR:-/tmp}/epm-docker-test-results.XXXXXX")" result_dir="$(mktemp -d "${TMPDIR:-/tmp}/epm-docker-test-results.XXXXXX")"
prepare_parallel_runner
if ((PARALLEL_JOBS > 0)); then if ((PARALLEL_JOBS > 0)); then
info "Running $total tests in parallel (max $PARALLEL_JOBS)" info "Running $total tests in parallel (max $PARALLEL_JOBS)"
...@@ -1038,6 +1106,7 @@ main() { ...@@ -1038,6 +1106,7 @@ main() {
else else
info "Live output is suppressed in parallel mode; logs will be saved to files" info "Live output is suppressed in parallel mode; logs will be saved to files"
fi fi
prepare_parallel_remote_source
for system_image in "${TARGET_SYSTEMS[@]}"; do for system_image in "${TARGET_SYSTEMS[@]}"; do
((index += 1)) ((index += 1))
...@@ -1051,6 +1120,7 @@ main() { ...@@ -1051,6 +1120,7 @@ main() {
done done
wait wait
cleanup_parallel_remote_source
for ((i = 1; i <= total; i++)); do for ((i = 1; i <= total; i++)); do
if [[ -f "$result_dir/$i.result" ]]; then if [[ -f "$result_dir/$i.result" ]]; then
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment