f-80: e2e: verdicts propres à chaque test, API et journal injoignables en échec, nettoyage des fichiers t<NN>, sélection stricte des tests #80

Signed-off-by: GnomeZworc <nicolas.boufidjeline@g3e.fr>
This commit is contained in:
GnomeZworc 2026-10-23 13:48:22 +02:00
commit ddd20a88c5
Signed by: nicolas.boufideline
GPG key ID: 4406BBBF8845D632
13 changed files with 203 additions and 50 deletions

View file

@ -10,7 +10,8 @@ usage () {
usage: ${0##*/} [--clean] usage: ${0##*/} [--clean]
sans option : compile ${BINARIES} pour linux/amd64 depuis l'arbre courant dans ${BIN_DIR} ; sans option : compile ${BINARIES} pour linux/amd64 depuis l'arbre courant dans ${BIN_DIR} ;
le prochain lab les installe par-dessus la release de la topologie le prochain lab les installe par-dessus la release épinglée dans
topologies/provision/hypervisor.sh
--clean : vide ${BIN_DIR} ; le prochain lab teste la release seule --clean : vide ${BIN_DIR} ; le prochain lab teste la release seule
USAGE USAGE
exit 2 exit 2

View file

@ -55,15 +55,18 @@ api_state () {
} }
wait_state () { wait_state () {
local path="${1}" want="${2}" i s local path="${1}" want="${2}" i s code=""
for i in $(seq 1 180); do for i in $(seq 1 180); do
s=$(api_state "${path}") s=$(api_state "${path}")
[ "${s}" = "${want}" ] && return 0 [ "${s}" = "${want}" ] && return 0
[ "${want}" = deleted ] && [ -z "${s}" ] && return 0 if [ "${want}" = deleted ] && [ -z "${s}" ]; then
code=$(api_code GET "${path}")
[ "${code}" = 404 ] && return 0
fi
[ "${s}" = error ] && { echo "${path}: error" >&2; return 1; } [ "${s}" = error ] && { echo "${path}: error" >&2; return 1; }
sleep 2 sleep 2
done done
echo "${path}: ${s:-absent}, ${want} attendu" >&2 echo "${path}: ${s:-absent}${code:+ (HTTP ${code})}, ${want} attendu" >&2
return 1 return 1
} }

View file

@ -76,20 +76,24 @@ wait_watchdog () {
} }
watchdog_alerts () { watchdog_alerts () {
journalctl -u agent --since "-${1:-300}s" --no-pager -o cat | grep -F 'watchdog: inconsistency detected' local log
log=$(journalctl -u agent --since "-${1:-300}s" --no-pager -o cat) || { echo "journal de l'agent illisible (journalctl en échec)"; return 1; }
grep -F 'watchdog: inconsistency detected' <<<"${log}" || true
} }
watchdog_quiet () { watchdog_quiet () {
local out local alerts out
out=$(watchdog_alerts "${1:-300}" | { if [ -n "${2:-}" ]; then grep -vF -- "${2}"; else cat; fi; }) alerts=$(watchdog_alerts "${1:-300}") || { printf '%s\n' "${alerts}"; return 1; }
out=$(printf '%s\n' "${alerts}" | { if [ -n "${2:-}" ]; then grep -vF -- "${2}"; else cat; fi; })
[ -z "${out}" ] && return 0 [ -z "${out}" ] && return 0
printf '%s\n' "${out}" printf '%s\n' "${out}"
return 1 return 1
} }
watchdog_quiet_about () { watchdog_quiet_about () {
local out local alerts out
out=$(watchdog_alerts "${1}" | grep -F -- "${2}") || return 0 alerts=$(watchdog_alerts "${1}") || { printf '%s\n' "${alerts}"; return 1; }
out=$(grep -F -- "${2}" <<<"${alerts}") || return 0
printf '%s\n' "${out}" printf '%s\n' "${out}"
return 1 return 1
} }
@ -129,8 +133,9 @@ wait_bgp_paths () {
} }
watchdog_l3_quiet () { watchdog_l3_quiet () {
local out local alerts out
out=$(journalctl -u agent --since "-${1:-60}s" --no-pager -o cat | grep -F 'watchdog: inconsistency detected' | grep -E '"kind":"bgp"|l3vni [0-9]|interface (vrf|brl3|vxl3|l3)-') || return 0 alerts=$(watchdog_alerts "${1:-60}") || { printf '%s\n' "${alerts}"; return 1; }
out=$(grep -E '"kind":"bgp"|l3vni [0-9]|interface (vrf|brl3|vxl3|l3)-' <<<"${alerts}") || return 0
printf '%s\n' "${out}" printf '%s\n' "${out}"
return 1 return 1
} }

View file

@ -1,26 +1,52 @@
STATE_DIR=/var/tmp/e2e STATE_DIR=/var/tmp/e2e
SCRATCH_DIR=/tmp
owned () { owned () {
grep -E "^(vp-|sn-)?t${T}" grep -E "^(vp-|sn-)?t${T}"
} }
api_names () { api_names () {
curl -fsS "${API}/${1}" | jq -r '.[] | select(.state != "deleted") | .name' local out
out=$(curl -fsS "${API}/${1}") || { echo "API injoignable (${1})" >&2; return 1; }
jq -r '.[] | select(.state != "deleted") | .name' <<<"${out}"
} }
delete_owned () { delete_owned () {
local kind="${1}" name rc=0 local kind="${1}" names name rc=0
for name in $(api_names "${kind}" | owned); do names=$(api_names "${kind}") || return 1
for name in $(owned <<<"${names}"); do
api_delete "${kind}/${name}" && wait_state "${kind}/${name}" deleted || { echo "${kind}/${name} : suppression en échec" >&2; rc=1; } api_delete "${kind}/${name}" && wait_state "${kind}/${name}" deleted || { echo "${kind}/${name} : suppression en échec" >&2; rc=1; }
done done
return "${rc}" return "${rc}"
} }
scratch_netns () {
ip netns list | awk '{print $1}' | grep -E "^t${T}-"
}
scratch_files () {
find "${SCRATCH_DIR}" "${STATE_DIR}" -maxdepth 1 -type f \( -name "t${T}-*" -o -name "t${T}.*" \) ! -name "t${T}.avant"
}
clean_scratch () {
local ns f
for ns in $(scratch_netns); do
ip netns del "${ns}" || return 1
done
for f in $(scratch_files); do
rm -f "${f}" || return 1
done
}
teardown_owned () { teardown_owned () {
delete_owned vms && delete_owned subnets && delete_owned vpcs || return 1 delete_owned vms && delete_owned subnets && delete_owned vpcs || return 1
rm -f "${VOLUMES}/t${T}"-* rm -f "${VOLUMES}/t${T}"-*
local left clean_scratch || return 1
left=$(for kind in vms subnets vpcs; do api_names "${kind}" | owned | sed "s|^|${kind}/|"; done | tr '\n' ' ') local left="" kind names
for kind in vms subnets vpcs; do
names=$(api_names "${kind}") || return 1
left="${left}$(owned <<<"${names}" | sed "s|^|${kind}/|" | tr '\n' ' ')"
done
[ -z "${left}" ] || { echo "encore présents : ${left}"; return 1; } [ -z "${left}" ] || { echo "encore présents : ${left}"; return 1; }
} }
@ -40,8 +66,9 @@ host_state () {
prepare_test () { prepare_test () {
mkdir -p "${STATE_DIR}" mkdir -p "${STATE_DIR}"
if [ -n "$(for kind in vms subnets vpcs; do api_names "${kind}" | owned; done)" ]; then curl -fsS -o /dev/null "${API}/vpcs" || echo "ÉCHOUÉ: ${HOST} : API de l'agent injoignable avant le test"
info "${HOST} : ressources t${T} d'un passage précédent, suppression" if [ -n "$(for kind in vms subnets vpcs; do api_names "${kind}" | owned; done)$(scratch_netns)$(scratch_files)" ]; then
info "${HOST} : ressources, netns ou fichiers t${T} d'un passage précédent, suppression"
check "${HOST} : nettoyage des ressources t${T} restantes" teardown_owned check "${HOST} : nettoyage des ressources t${T} restantes" teardown_owned
fi fi
host_state > "${STATE_DIR}/t${T}.avant" host_state > "${STATE_DIR}/t${T}.avant"

View file

@ -25,6 +25,11 @@ two_image () {
rm -rf "${work}" rm -rf "${work}"
} }
vm_volume () {
two_image || { echo "image ${TWO_IMAGE} indisponible" >&2; return 1; }
mkdir -p "${VOLUMES}" && cp "${TWO_IMAGE}" "${VOLUMES}/${1}.qcow2"
}
vm_key () { vm_key () {
[ -f "${VM_KEY}" ] || ssh-keygen -q -t ed25519 -N '' -C lab-vm -f "${VM_KEY}" [ -f "${VM_KEY}" ] || ssh-keygen -q -t ed25519 -N '' -C lab-vm -f "${VM_KEY}"
cat "${VM_KEY}.pub" cat "${VM_KEY}.pub"
@ -40,8 +45,7 @@ vm_create () {
local name="${1}" subnet="${2}" ip="${3}" key="${4:-}" meta="{}" local name="${1}" subnet="${2}" ip="${3}" key="${4:-}" meta="{}"
api_state "vms/${name}" | grep -qx running && return 0 api_state "vms/${name}" | grep -qx running && return 0
[ -n "${key}" ] && meta="{\"sshkey\":\"${key}\"}" [ -n "${key}" ] && meta="{\"sshkey\":\"${key}\"}"
mkdir -p "${VOLUMES}" vm_volume "${name}" || return 1
cp "${TWO_IMAGE}" "${VOLUMES}/${name}.qcow2" || return 1
api_post vms "{\"name\":\"${name}\",\"memory\":1024,\"cpus\":1,\"metadata\":${meta},\"interfaces\":[{\"subnet\":\"${subnet}\",\"ip\":\"${ip}\",\"mac\":\"$(mac_for_ip "${ip}")\",\"primary\":true}],\"storage\":[{\"path\":\"${VOLUMES}/${name}.qcow2\",\"dev\":\"vda\"}]}" \ api_post vms "{\"name\":\"${name}\",\"memory\":1024,\"cpus\":1,\"metadata\":${meta},\"interfaces\":[{\"subnet\":\"${subnet}\",\"ip\":\"${ip}\",\"mac\":\"$(mac_for_ip "${ip}")\",\"primary\":true}],\"storage\":[{\"path\":\"${VOLUMES}/${name}.qcow2\",\"dev\":\"vda\"}]}" \
&& wait_state "vms/${name}" running && wait_state "vms/${name}" running
} }
@ -77,6 +81,14 @@ vm_fails () {
esac esac
} }
vm_output_empty () {
local out
out=$(vm_ssh "${1}" "${2}" "${3}") || { echo "commande en échec dans ${2} (ou SSH en panne)"; return 1; }
[ -z "${out}" ] && return 0
echo "sortie non vide dans ${2} : $(printf '%s' "${out}" | head -n 1)"
return 1
}
vm_grep () { vm_grep () {
vm_ssh "${1}" "${2}" "${3} 2>&1" | grep -q -- "${4}" vm_ssh "${1}" "${2}" "${3} 2>&1" | grep -q -- "${4}"
} }
@ -125,8 +137,7 @@ vm_launch () {
shift shift
api_state "vms/${name}" | grep -qx running && return 0 api_state "vms/${name}" | grep -qx running && return 0
body=$(vm_json "${name}" --disk "vda=${VOLUMES}/${name}.qcow2" "$@") || return 1 body=$(vm_json "${name}" --disk "vda=${VOLUMES}/${name}.qcow2" "$@") || return 1
mkdir -p "${VOLUMES}" vm_volume "${name}" || return 1
cp "${TWO_IMAGE}" "${VOLUMES}/${name}.qcow2" || return 1
api_post vms "${body}" && wait_state "vms/${name}" running api_post vms "${body}" && wait_state "vms/${name}" running
} }

View file

@ -15,10 +15,12 @@ usage: ${0##*/} [--continue] [--connus] all | --from <NN> | <couche N0>... | <te
tests : $(cd "${SUITE_DIR}" && ls -- *.sh 2>/dev/null | sed 's/\.sh$//' | tr '\n' ' ') tests : $(cd "${SUITE_DIR}" && ls -- *.sh 2>/dev/null | sed 's/\.sh$//' | tr '\n' ' ')
Chaque test s'exécute depuis le Mac sur le lab en cours (labctl.sh up, push, ./labctl up). Chaque test s'exécute depuis le Mac sur le lab en cours (labctl.sh up, push, ./labctl up).
Une couche se désigne par son numéro (00, 10, …, 70), un test par son numéro ou son nom. Une couche se désigne par son numéro (00, 10, …, 70), un test par son numéro à deux chiffres
Avant un test, ses ressources t<NN> restantes sont supprimées et l'état de chaque hôte de ou son nom exact, avec ou sans numéro (21, 21-dhcp-two ou dhcp-two).
l'en-tête « # hôtes: » est relevé ; après, ses ressources sont supprimées et l'hôte doit être Avant un test, ses ressources t<NN> restantes (API, netns t<NN>-*, fichiers t<NN>-* et t<NN>.*
revenu à cet état. Avec all ou --from, une couche en échec arrête la suite (--continue : non). de /tmp et /var/tmp/e2e) sont supprimées et l'état de chaque hôte de l'en-tête « # hôtes: »
est relevé ; après, ses ressources sont supprimées et l'hôte doit être revenu à cet état.
Un test réussit s'il n'a aucun échec et au moins une vérification propre, hors fixtures. Avec all ou --from, une couche en échec arrête la suite (--continue : non).
Un contrôle marqué known (échec connu, rattaché à un ticket) donne CONNU sans compter comme Un contrôle marqué known (échec connu, rattaché à un ticket) donne CONNU sans compter comme
un échec, ou INATTENDU (compté comme un échec) s'il passe : retirer alors le marqueur. --connus un échec, ou INATTENDU (compté comme un échec) s'il passe : retirer alors le marqueur. --connus
compte les CONNU comme des échecs. known_flaky (échec intermittent) donne RÉUSSI, annoté, s'il passe. compte les CONNU comme des échecs. known_flaky (échec intermittent) donne RÉUSSI, annoté, s'il passe.
@ -53,7 +55,7 @@ number () {
} }
run_scenario () { run_scenario () {
local file="${1}" name passed failed known host hosts num local file="${1}" name passed failed known host hosts num start own
name="$(basename "${file}" .sh)" name="$(basename "${file}" .sh)"
num=$(number "${file}") num=$(number "${file}")
hosts=$(header "${file}" hôtes) hosts=$(header "${file}" hôtes)
@ -62,7 +64,9 @@ run_scenario () {
for host in ${hosts}; do for host in ${hosts}; do
on "${host}" "T=${num}" "HOST=${host}" <<< 'prepare_test' on "${host}" "T=${num}" "HOST=${host}" <<< 'prepare_test'
done done
start=$(wc -l < "${SCENARIO_LOG}")
( . "${file}" ) ( . "${file}" )
own=$(tail -n "+$(( start + 1 ))" "${SCENARIO_LOG}" | grep -cE '^(RÉUSSI|ÉCHOUÉ|CONNU|INATTENDU): ')
for host in ${hosts}; do for host in ${hosts}; do
on "${host}" "T=${num}" "HOST=${host}" <<< 'finish_test' on "${host}" "T=${num}" "HOST=${host}" <<< 'finish_test'
done done
@ -72,7 +76,8 @@ run_scenario () {
[[ "${KNOWN_FAIL}" -eq 1 ]] && failed=$(( failed + known )) [[ "${KNOWN_FAIL}" -eq 1 ]] && failed=$(( failed + known ))
rm -f "${SCENARIO_LOG}" rm -f "${SCENARIO_LOG}"
echo "=== ${name} : ${passed} réussi(s), ${failed} échoué(s)$([[ "${known}" -gt 0 ]] && echo ", ${known} connu(s)")" echo "=== ${name} : ${passed} réussi(s), ${failed} échoué(s)$([[ "${known}" -gt 0 ]] && echo ", ${known} connu(s)")"
[[ "${failed}" -eq 0 && "${passed}" -gt 0 ]] [[ "${own}" -gt 0 ]] || echo "=== ${name} : aucune vérification propre au scénario n'a tourné"
[[ "${failed}" -eq 0 && "${own}" -gt 0 ]]
} }
matrix () { matrix () {
@ -100,10 +105,16 @@ select_files () {
ls "${SUITE_DIR}"/*.sh ls "${SUITE_DIR}"/*.sh
elif [[ "${arg}" =~ ^[0-9]0$ ]]; then elif [[ "${arg}" =~ ^[0-9]0$ ]]; then
ls "${SUITE_DIR}/${arg:0:1}"[0-9]-*.sh 2>/dev/null || { echo "couche vide : ${arg}" >&2; return 1; } ls "${SUITE_DIR}/${arg:0:1}"[0-9]-*.sh 2>/dev/null || { echo "couche vide : ${arg}" >&2; return 1; }
else elif [[ "${arg}" =~ ^[0-9]{2}$ ]]; then
file=$(ls "${SUITE_DIR}/${arg}"*.sh 2>/dev/null | head -n 1) file=$(ls "${SUITE_DIR}/${arg}"-*.sh 2>/dev/null | head -n 1)
[[ -n "${file}" ]] || { echo "test inconnu : ${arg}" >&2; return 1; } [[ -n "${file}" ]] || { echo "test inconnu : ${arg}" >&2; return 1; }
echo "${file}" echo "${file}"
elif [[ -f "${SUITE_DIR}/${arg}.sh" ]]; then
echo "${SUITE_DIR}/${arg}.sh"
else
file=$(ls "${SUITE_DIR}"/[0-9][0-9]-"${arg}".sh 2>/dev/null | head -n 1)
[[ -n "${file}" ]] || { echo "test inconnu : ${arg} (numéro à deux chiffres ou nom exact)" >&2; return 1; }
echo "${file}"
fi fi
done done
[[ -z "${from}" ]] || { echo "--from sans numéro" >&2; return 1; } [[ -z "${from}" ]] || { echo "--from sans numéro" >&2; return 1; }

View file

@ -20,7 +20,11 @@ if ! bash -n "${FAKE_DIR}/payload.$(( N + 1 ))" 2>"${FAKE_DIR}/syntax.$(( N + 1
echo "ÉCHOUÉ: syntaxe du bloc $(( N + 1 ))" echo "ÉCHOUÉ: syntaxe du bloc $(( N + 1 ))"
exit 0 exit 0
fi fi
[ -n "${FAKE_OUT:-}" ] && printf '%b\n' "${FAKE_OUT}" case "$(tail -n 1 "${FAKE_DIR}/payload.$(( N + 1 ))")" in
prepare_test|finish_test) OUT="${FAKE_FIXTURE_OUT:-${FAKE_OUT:-}}" ;;
*) OUT="${FAKE_OUT:-}" ;;
esac
[ -n "${OUT}" ] && printf '%b\n' "${OUT}"
exit 0 exit 0
FAKE FAKE
chmod +x "${WORK}/lab-host" chmod +x "${WORK}/lab-host"
@ -29,7 +33,7 @@ FAKE
run () { run () {
env LAB_HOST="${WORK}/lab-host" FAKE_DIR="${WORK}" SUITE_DIR="${SUITE_DIR:-${WORK}/scenarios}" \ env LAB_HOST="${WORK}/lab-host" FAKE_DIR="${WORK}" SUITE_DIR="${SUITE_DIR:-${WORK}/scenarios}" \
FAKE_OUT="${FAKE_OUT:-}" bash "${DIR}/run.sh" "$@" > "${WORK}/out.log" 2>&1 FAKE_OUT="${FAKE_OUT:-}" FAKE_FIXTURE_OUT="${FAKE_FIXTURE_OUT:-}" bash "${DIR}/run.sh" "$@" > "${WORK}/out.log" 2>&1
} }
teardown () { teardown () {
@ -39,7 +43,7 @@ teardown () {
} }
node () { node () {
bash -c "$(printf ". '%s'; " "${DIR}"/lib/{core,vm,net,evpn,fixture}.sh)${1}" 2>&1 bash -c "$(printf ". '%s'; " "${DIR}"/lib/{core,vm,net,evpn,fixture}.sh)two_image () { echo 'two_image non simulée' >&2; return 1; }; ${1}" 2>&1
} }
test_on_sends_assignments_library_and_block_to_the_node () { test_on_sends_assignments_library_and_block_to_the_node () {
@ -135,7 +139,84 @@ test_wait_state () {
local start=${SECONDS} local start=${SECONDS}
[[ $(node 'api_state () { echo error; }; wait_state vms/x running 2>&1 || echo NON') == *NON ]] || fail "error accepté" [[ $(node 'api_state () { echo error; }; wait_state vms/x running 2>&1 || echo NON') == *NON ]] || fail "error accepté"
(( SECONDS - start < 5 )) || fail "une ressource en error n'arrête pas l'attente immédiatement ($(( SECONDS - start )) s)" (( SECONDS - start < 5 )) || fail "une ressource en error n'arrête pas l'attente immédiatement ($(( SECONDS - start )) s)"
[[ $(node 'api_state () { echo; }; wait_state vms/x deleted && echo OUI') == OUI ]] || fail "absence = supprimé" [[ $(node 'api_state () { echo; }; api_code () { echo 404; }; wait_state vms/x deleted && echo OUI') == OUI ]] || fail "absence (404) = supprimé"
[[ $(node 'api_state () { echo; }; api_code () { echo 000; }; sleep () { :; }; wait_state vms/x deleted 2>&1 || echo NON') == *'(HTTP 000), deleted attendu'*NON ]] \
|| fail "API injoignable prise pour une suppression : $(node 'api_state () { echo; }; api_code () { echo 000; }; sleep () { :; }; wait_state vms/x deleted 2>&1 || echo NON')"
teardown
}
test_api_names_and_teardown_fail_when_the_api_is_unreachable () {
setup "api_names, delete_owned, teardown_owned : échec si l'API est injoignable"
local stub='curl () { return 7; }; scratch_netns () { :; }; scratch_files () { :; }; rm () { :; }; '
[[ $(node "${stub}api_names vms 2>&1 || echo NON") == *'API injoignable (vms)'*NON ]] || fail "api_names : $(node "${stub}api_names vms 2>&1 || echo NON")"
[[ $(node "T=41; ${stub}delete_owned vms 2>&1 || echo NON") == *NON ]] || fail "delete_owned réussit sans API"
[[ $(node "T=41; ${stub}teardown_owned 2>&1 || echo NON") == *NON ]] || fail "teardown_owned réussit sans API"
teardown
}
test_prepare_reports_an_unreachable_api () {
setup "prepare_test : API injoignable signalée en ÉCHOUÉ"
local dir
dir=$(mktemp -d)
[[ $(node "T=41; HOST=hv1; STATE_DIR='${dir}'; curl () { return 7; }; api_names () { return 1; }; scratch_netns () { :; }; scratch_files () { :; }; host_state () { :; }; prepare_test") == "ÉCHOUÉ: hv1 : API de l'agent injoignable avant le test" ]] \
|| fail "$(node "T=41; HOST=hv1; STATE_DIR='${dir}'; curl () { return 7; }; api_names () { return 1; }; scratch_netns () { :; }; scratch_files () { :; }; host_state () { :; }; prepare_test")"
rm -rf "${dir}"
teardown
}
test_clean_scratch_removes_only_the_test_netns_and_files () {
setup "clean_scratch : netns et fichiers t<NN> du test, rien d'autre, état d'avant gardé"
local dir out
dir=$(mktemp -d)
mkdir "${dir}/tmp" "${dir}/state"
touch "${dir}/tmp/t41-a2.log" "${dir}/tmp/t41.log" "${dir}/tmp/t41.10.255.255.1" "${dir}/tmp/t411.log" "${dir}/tmp/t42-a.log" \
"${dir}/state/t41.restart" "${dir}/state/t41.avant" "${dir}/state/t42.avant"
out=$(node "T=41; SCRATCH_DIR='${dir}/tmp'; STATE_DIR='${dir}/state'; ip () { if [ \"\$1 \$2\" = 'netns list' ]; then printf '%s\n' 't41-ebt (id: 3)' vp-t41a t42-ebt; else echo \"ip \$*\"; fi; }; clean_scratch && echo OUI")
[[ $(tr '\n' '|' <<< "${out}") == "ip netns del t41-ebt|OUI|" ]] || fail "netns : ${out}"
[[ $(cd "${dir}" && find . -type f | sort | tr '\n' ' ') == "./state/t41.avant ./state/t42.avant ./tmp/t411.log ./tmp/t42-a.log " ]] \
|| fail "fichiers restants : $(cd "${dir}" && find . -type f | sort | tr '\n' ' ')"
rm -rf "${dir}"
teardown
}
test_scenario_without_its_own_check_fails_despite_fixtures () {
setup "un scénario sans vérification propre échoue, même si les fixtures réussissent"
suite_file 41-x hv1
FAKE_FIXTURE_OUT='RÉUSSI: teardown\nRÉUSSI: assert_clean' FAKE_OUT='INFO: rien' run 41; [[ $? -eq 1 ]] || fail "scénario muet accepté"
grep -q "aucune vérification propre au scénario" "${WORK}/out.log" || fail "cause non annoncée"
FAKE_FIXTURE_OUT='RÉUSSI: teardown' FAKE_OUT='CONNU: x (#1)' run 41; [[ $? -eq 0 ]] || fail "un CONNU propre au scénario n'est pas compté comme une vérification"
FAKE_FIXTURE_OUT='INFO: rien' FAKE_OUT='RÉUSSI: x' run 41; [[ $? -eq 0 ]] || fail "vérification propre non comptée"
teardown
}
test_selection_requires_two_digits_or_an_exact_name () {
setup "sélection : numéro à deux chiffres ou nom exact, sans préfixe partiel"
suite_file 21-dhcp-two ""; suite_file 41-x ""; suite_file 42-y ""
run 4; [[ $? -eq 2 ]] || fail "« 4 » accepté"
run dhcp; [[ $? -eq 2 ]] || fail "« dhcp » accepté"
FAKE_OUT='RÉUSSI: x' run dhcp-two
[[ $(grep '^=== [0-9a-z-]*$' "${WORK}/out.log" | tr '\n' ' ') == "=== 21-dhcp-two " ]] || fail "par nom : $(grep '^=== ' "${WORK}/out.log" | tr '\n' ' ')"
FAKE_OUT='RÉUSSI: x' run 21-dhcp-two 42
[[ $(grep '^=== [0-9a-z-]*$' "${WORK}/out.log" | tr '\n' ' ') == "=== 21-dhcp-two === 42-y " ]] || fail "nom complet et numéro : $(grep '^=== ' "${WORK}/out.log" | tr '\n' ' ')"
teardown
}
test_vm_volume_builds_the_image_before_copying () {
setup "vm_volume : image construite si absente, échec sans image"
local dir
dir=$(mktemp -d)
[[ $(node "VOLUMES='${dir}/v'; TWO_IMAGE='${dir}/img'; two_image () { echo base > \"\${TWO_IMAGE}\"; }; vm_volume t41-a && cat '${dir}/v/t41-a.qcow2'") == base ]] || fail "image non construite ou non copiée"
[[ $(node "VOLUMES='${dir}/w'; TWO_IMAGE='${dir}/absente'; two_image () { return 1; }; vm_volume t41-a 2>&1 || echo NON") == *NON ]] || fail "volume sans image accepté"
[[ ! -e "${dir}/w/t41-a.qcow2" ]] || fail "volume créé sans image"
rm -rf "${dir}"
teardown
}
test_vm_output_empty_needs_a_working_command () {
setup "vm_output_empty : sortie vide d'une commande réussie, SSH ou commande en échec refusés"
[[ $(node 'vm_ssh () { :; }; vm_output_empty v 1.2.3.4 x && echo OUI') == OUI ]] || fail "sortie vide refusée"
[[ $(node 'vm_ssh () { echo "10.33.0.0/16 via 10.33.0.1"; }; vm_output_empty v 1.2.3.4 x || echo NON') == *NON ]] || fail "sortie non vide acceptée"
[[ $(node 'vm_ssh () { return 255; }; vm_output_empty v 1.2.3.4 x || echo NON') == *NON ]] || fail "SSH en panne pris pour une sortie vide"
teardown teardown
} }
@ -240,6 +321,13 @@ test_watchdog_quiet_passes_without_alert_and_fails_on_one () {
[[ $(node "journalctl () { echo '${alert}'; }; watchdog_quiet 300 >/dev/null || echo NON") == NON ]] || fail "alerte acceptée sans exclusion" [[ $(node "journalctl () { echo '${alert}'; }; watchdog_quiet 300 >/dev/null || echo NON") == NON ]] || fail "alerte acceptée sans exclusion"
[[ $(node "journalctl () { echo '${alert}'; }; watchdog_quiet 300 'stale dhcp' && echo OUI") == OUI ]] || fail "alerte exclue refusée" [[ $(node "journalctl () { echo '${alert}'; }; watchdog_quiet 300 'stale dhcp' && echo OUI") == OUI ]] || fail "alerte exclue refusée"
teardown teardown
setup "watchdog_quiet, watchdog_quiet_about, watchdog_l3_quiet : échec si le journal est illisible"
local f
for f in 'watchdog_quiet 300' "watchdog_quiet 300 'stale dhcp'" "watchdog_quiet_about 300 'stale dhcp'" 'watchdog_l3_quiet 60'; do
[[ $(node "journalctl () { return 1; }; ${f} || echo NON") == *'journalctl en échec'*NON ]] || fail "${f} : $(node "journalctl () { return 1; }; ${f} || echo NON")"
done
[[ $(node "journalctl () { :; }; watchdog_quiet_about 300 'stale dhcp' && echo OUI") == OUI ]] || fail "watchdog_quiet_about : journal vide refusé"
teardown
} }
test_watchdog_reported_matches_only_watchdog_alerts () { test_watchdog_reported_matches_only_watchdog_alerts () {
@ -319,7 +407,7 @@ test_owned_keeps_only_the_resources_of_the_test () {
test_teardown_deletes_vms_then_subnets_then_vpcs () { test_teardown_deletes_vms_then_subnets_then_vpcs () {
setup "teardown_owned : VM, puis subnets, puis VPC, et rien d'autre" setup "teardown_owned : VM, puis subnets, puis VPC, et rien d'autre"
local stub='api_names () { case $1 in vms) [ -f /tmp/.t70.$$ ] || echo t41-a; echo t42-z ;; subnets) echo sn-t41a ;; vpcs) echo vp-t41a ;; esac; }; api_delete () { echo "DEL $1"; [ "$1" = vms/t41-a ] && touch /tmp/.t70.$$; :; }; wait_state () { :; }; rm () { :; }; ' local stub='api_names () { case $1 in vms) [ -f /tmp/.t70.$$ ] || echo t41-a; echo t42-z ;; subnets) echo sn-t41a ;; vpcs) echo vp-t41a ;; esac; }; api_delete () { echo "DEL $1"; [ "$1" = vms/t41-a ] && touch /tmp/.t70.$$; :; }; wait_state () { :; }; rm () { :; }; scratch_netns () { :; }; scratch_files () { :; }; '
local out local out
out=$(node "T=41; ${stub}teardown_owned; command rm -f /tmp/.t70.\$\$") out=$(node "T=41; ${stub}teardown_owned; command rm -f /tmp/.t70.\$\$")
[[ $(grep '^DEL' <<< "${out}" | tr '\n' ' ') == "DEL vms/t41-a DEL subnets/sn-t41a DEL vpcs/vp-t41a " ]] || fail "ordre : ${out}" [[ $(grep '^DEL' <<< "${out}" | tr '\n' ' ') == "DEL vms/t41-a DEL subnets/sn-t41a DEL vpcs/vp-t41a " ]] || fail "ordre : ${out}"
@ -384,7 +472,7 @@ test_vm_json_builds_nics_disks_metadata_and_uefi () {
test_vm_launch_copies_the_system_disk_first_and_waits () { test_vm_launch_copies_the_system_disk_first_and_waits () {
setup "vm_launch : disque système vda copié en premier, POST, attente de running" setup "vm_launch : disque système vda copié en premier, POST, attente de running"
local out local out
out=$(node 'VOLUMES=/v; TWO_IMAGE=/img; api_state () { echo; }; mkdir () { :; }; cp () { echo "CP $*"; }; api_post () { echo "POST $1 $(jq -c "[.storage[].dev, .interfaces[0].ip]" <<<"$2")"; }; wait_state () { echo "WAIT $*"; }; vm_launch t26-d --nic sn=10.0.0.10 --disk xvda=/v/s.qcow2') out=$(node 'VOLUMES=/v; TWO_IMAGE=/img; two_image () { :; }; api_state () { echo; }; mkdir () { :; }; cp () { echo "CP $*"; }; api_post () { echo "POST $1 $(jq -c "[.storage[].dev, .interfaces[0].ip]" <<<"$2")"; }; wait_state () { echo "WAIT $*"; }; vm_launch t26-d --nic sn=10.0.0.10 --disk xvda=/v/s.qcow2')
[[ "${out}" == $'CP /img /v/t26-d.qcow2\nPOST vms ["vda","xvda","10.0.0.10"]\nWAIT vms/t26-d running' ]] || fail "${out}" [[ "${out}" == $'CP /img /v/t26-d.qcow2\nPOST vms ["vda","xvda","10.0.0.10"]\nWAIT vms/t26-d running' ]] || fail "${out}"
[[ $(node 'api_state () { echo running; }; cp () { echo CP; }; api_post () { echo POST; }; vm_launch t --nic s=1.2.3.4 && echo OK') == OK ]] || fail "VM en running relancée" [[ $(node 'api_state () { echo running; }; cp () { echo CP; }; api_post () { echo POST; }; vm_launch t --nic s=1.2.3.4 && echo OK') == OK ]] || fail "VM en running relancée"
[[ $(node 'api_state () { echo; }; cp () { echo CP; }; vm_launch t --bad x || echo NON') == *NON ]] || fail "option invalide : disque copié ou POST envoyé" [[ $(node 'api_state () { echo; }; cp () { echo CP; }; vm_launch t --bad x || echo NON') == *NON ]] || fail "option invalide : disque copié ou POST envoyé"

View file

@ -20,7 +20,7 @@ esac
on "${host}" "HOST=${host}" "ADDR=${addr}" "BACKEND=${backend}" "SW1=${SW1}" "SW2=${SW2}" <<'NODE' on "${host}" "HOST=${host}" "ADDR=${addr}" "BACKEND=${backend}" "SW1=${SW1}" "SW2=${SW2}" <<'NODE'
check "${HOST} : agent actif" systemctl is-active --quiet agent check "${HOST} : agent actif" systemctl is-active --quiet agent
check "${HOST} : API de l'agent joignable" curl -fsS -o /dev/null "${API}/vpcs" check "${HOST} : API de l'agent joignable" curl -fsS -o /dev/null "${API}/vpcs"
check "${HOST} : backend DHCP ${BACKEND} configuré" grep -A1 -x 'dhcp:' /etc/two/agent.yml | grep -qx " backend: ${BACKEND}" check "${HOST} : backend DHCP ${BACKEND} configuré" sh -c "grep -A1 -x 'dhcp:' /etc/two/agent.yml | grep -qx ' backend: ${BACKEND}'"
check "${HOST} : VTEP = adresse underlay ${ADDR}" test "$(vtep_host)" = "${ADDR}" check "${HOST} : VTEP = adresse underlay ${ADDR}" test "$(vtep_host)" = "${ADDR}"
for peer in 192.168.14.11 192.168.14.12 192.168.14.13; do for peer in 192.168.14.11 192.168.14.12 192.168.14.13; do
[ "${peer}" = "${ADDR}" ] && continue [ "${peer}" = "${ADDR}" ] && continue

View file

@ -45,7 +45,7 @@ for dev in sdb vdb xvdaa; do
done done
check "VM : user_data en base64 invalide, 400" expect_code 400 api_post_code vms "$(vm_body t11-x sn-t11a 10.11.1.12 "${NONE}" '{"user_data":"pas du base64 !"}')" check "VM : user_data en base64 invalide, 400" expect_code 400 api_post_code vms "$(vm_body t11-x sn-t11a 10.11.1.12 "${NONE}" '{"user_data":"pas du base64 !"}')"
check "VMs refusées : aucune trace de t11-x" expect_code 404 api_code GET vms/t11-x check "VMs refusées : aucune trace de t11-x" expect_code 404 api_code GET vms/t11-x
cp "${TWO_IMAGE}" "${VOLUMES}/t11-ud.qcow2" check "volume de t11-ud" vm_volume t11-ud
check "VM t11-ud avec user_data en base64 valide, 202 (témoin)" expect_code 202 api_post_code vms "$(vm_body t11-ud sn-t11a 10.11.1.12 "" "{\"user_data\":\"$(printf '#cloud-config\n' | base64 | tr -d '\n')\"}")" check "VM t11-ud avec user_data en base64 valide, 202 (témoin)" expect_code 202 api_post_code vms "$(vm_body t11-ud sn-t11a 10.11.1.12 "" "{\"user_data\":\"$(printf '#cloud-config\n' | base64 | tr -d '\n')\"}")"
check "t11-ud running" wait_state vms/t11-ud running check "t11-ud running" wait_state vms/t11-ud running
NODE NODE

View file

@ -4,8 +4,7 @@
on hv1 <<'NODE' on hv1 <<'NODE'
check "VPC vp-t12a" vpc_create vp-t12a 10.12.0.0/16 check "VPC vp-t12a" vpc_create vp-t12a 10.12.0.0/16
mkdir -p "${VOLUMES}" check "volume de t12-a" vm_volume t12-a
cp "${TWO_IMAGE}" "${VOLUMES}/t12-a.qcow2"
post_both () { api_post subnets "${1}" && api_post vms "${2}"; } post_both () { api_post subnets "${1}" && api_post vms "${2}"; }
check "subnet sn-t12a et VM t12-a envoyés sans attendre" post_both \ check "subnet sn-t12a et VM t12-a envoyés sans attendre" post_both \
'{"name":"sn-t12a","vpc":"vp-t12a","mode":"vxlan","vxlan_id":1201,"iface_type":"vms","cidr":"10.12.1.0/24"}' \ '{"name":"sn-t12a","vpc":"vp-t12a","mode":"vxlan","vxlan_id":1201,"iface_type":"vms","cidr":"10.12.1.0/24"}' \
@ -37,6 +36,6 @@ check "sn-t12a recréé sous le même nom" subnet_create sn-t12a vp-t12a 1201 10
check "suppression de sn-t12a recréé" expect_code 202 api_code DELETE subnets/sn-t12a check "suppression de sn-t12a recréé" expect_code 202 api_code DELETE subnets/sn-t12a
check "sn-t12a recréé puis supprimé" wait_state subnets/sn-t12a deleted check "sn-t12a recréé puis supprimé" wait_state subnets/sn-t12a deleted
check "suppression de vp-t12a, sans subnet, 202 (témoin)" expect_code 202 api_code DELETE vpcs/vp-t12a check "suppression de vp-t12a, sans subnet, 202 (témoin)" expect_code 202 api_code DELETE vpcs/vp-t12a
check "vp-t12a supprimée" wait_state vpcs/vp-t12a deleted check "vp-t12a supprimé" wait_state vpcs/vp-t12a deleted
check "vp-t12a recréée sous le même nom" vpc_create vp-t12a 10.12.0.0/16 check "vp-t12a recréé sous le même nom" vpc_create vp-t12a 10.12.0.0/16
NODE NODE

View file

@ -8,6 +8,12 @@ HASH=$(openssl passwd -6 -salt t23lab two-lab)
UD=$(printf '%s\n' '#cloud-config' 'runcmd:' " - [sh, -c, 'echo t23 > /var/tmp/t23-runcmd']") UD=$(printf '%s\n' '#cloud-config' 'runcmd:' " - [sh, -c, 'echo t23 > /var/tmp/t23-runcmd']")
UD64=$(printf '%s\n' "${UD}" | base64 -w0) UD64=$(printf '%s\n' "${UD}" | base64 -w0)
served () { cat "/run/two/metadata/${1}/${2}"; } served () { cat "/run/two/metadata/${1}/${2}"; }
served_empty () {
local f="/run/two/metadata/${1}/${2}"
[ -f "${f}" ] || { echo "${f} absent"; return 1; }
[ -s "${f}" ] && { echo "${f} non vide"; return 1; }
return 0
}
check "VPC vp-t23" vpc_create vp-t23 10.23.0.0/16 check "VPC vp-t23" vpc_create vp-t23 10.23.0.0/16
check "subnet sn-t23" subnet_create sn-t23 vp-t23 2301 10.23.1.0/24 check "subnet sn-t23" subnet_create sn-t23 vp-t23 2301 10.23.1.0/24
check "VM t23-k : sshkey, password et user_data" vm_launch t23-k --nic sn-t23=10.23.1.10 --sshkey "${KEY}" --password "${HASH}" --user-data "${UD64}" check "VM t23-k : sshkey, password et user_data" vm_launch t23-k --nic sn-t23=10.23.1.10 --sshkey "${KEY}" --password "${HASH}" --user-data "${UD64}"
@ -24,15 +30,16 @@ check "t23-k : hash du password posé sur le compte syonad" vm_ssh vp-t23 10.23.
check "t23-s joignable en SSH" vm_wait vp-t23 10.23.1.11 check "t23-s joignable en SSH" vm_wait vp-t23 10.23.1.11
check "t23-s : cloud-init terminé" vm_cloud_init_done vp-t23 10.23.1.11 check "t23-s : cloud-init terminé" vm_cloud_init_done vp-t23 10.23.1.11
check "t23-s : user_data absent, user-data servi vide (template)" test -z "$(vm_meta vp-t23 10.23.1.11 user-data)" check "t23-s : user_data absent, user-data servi vide (template)" vm_output_empty vp-t23 10.23.1.11 'curl -fsS http://169.254.169.254/user-data'
check "t23-s : aucun runcmd exécuté" vm_fails vp-t23 10.23.1.11 'test -e /var/tmp/t23-runcmd' check "t23-s : aucun runcmd exécuté" vm_fails vp-t23 10.23.1.11 'test -e /var/tmp/t23-runcmd'
check "t23-s : password absent, compte syonad verrouillé" vm_ssh vp-t23 10.23.1.11 "sudo grep -q '^syonad:!' /etc/shadow" check "t23-s : password absent, compte syonad verrouillé" vm_ssh vp-t23 10.23.1.11 "sudo grep -q '^syonad:!' /etc/shadow"
check "t23-s : vendor-data sans passwd" vm_fails vp-t23 10.23.1.11 'curl -fsS http://169.254.169.254/vendor-data | grep -qE "^ *passwd:"' check "t23-s : vendor-data servi, avec la clé SSH (témoin)" vm_grep vp-t23 10.23.1.11 'curl -fsS http://169.254.169.254/vendor-data' "$(cut -d" " -f2 <<< "${KEY}")"
check "t23-s : vendor-data sans passwd" vm_output_empty vp-t23 10.23.1.11 'v=$(curl -fsS http://169.254.169.254/vendor-data) || exit 1; printf "%s\n" "${v}" | grep -E "^ *passwd:" || true'
served_line () { served "${1}" "${2}" | grep -qx -- "${3}"; } served_line () { served "${1}" "${2}" | grep -qx -- "${3}"; }
check "t23-n : serveur de métadonnées actif (metadata@t23-n)" systemctl is-active --quiet metadata@t23-n.service check "t23-n : serveur de métadonnées actif (metadata@t23-n)" systemctl is-active --quiet metadata@t23-n.service
check "t23-n : meta-data préparé (témoin)" served_line t23-n meta-data 'instance-id: t23-n' check "t23-n : meta-data préparé (témoin)" served_line t23-n meta-data 'instance-id: t23-n'
check "t23-n : vendor-data vide, aucun compte à créer" test -z "$(served t23-n vendor-data)" check "t23-n : vendor-data vide, aucun compte à créer" served_empty t23-n vendor-data
check "t23-n : user-data vide" test -z "$(served t23-n user-data)" check "t23-n : user-data vide" served_empty t23-n user-data
check "t23-n : démarrée, SSH refusé faute de compte (Permission denied)" ssh_denied vp-t23 10.23.1.12 check "t23-n : démarrée, SSH refusé faute de compte (Permission denied)" ssh_denied vp-t23 10.23.1.12
NODE NODE

View file

@ -21,7 +21,7 @@ for ip in 10.33.0.21 10.33.0.22; do
check "${ip} : bail servi par interface_ip 10.33.0.1 (server-id)" test "$(vm_lease vp-t33 "${ip}" SERVER_ADDRESS)" = 10.33.0.1 check "${ip} : bail servi par interface_ip 10.33.0.1 (server-id)" test "$(vm_lease vp-t33 "${ip}" SERVER_ADDRESS)" = 10.33.0.1
check "${ip} : route par défaut via 10.33.0.1" route_via vp-t33 "${ip}" default 10.33.0.1 check "${ip} : route par défaut via 10.33.0.1" route_via vp-t33 "${ip}" default 10.33.0.1
check "${ip} : route /32 vers les métadonnées via 10.33.0.1" route_via vp-t33 "${ip}" 169.254.169.254 10.33.0.1 check "${ip} : route /32 vers les métadonnées via 10.33.0.1" route_via vp-t33 "${ip}" 169.254.169.254 10.33.0.1
check "${ip} : pas de route vers le VPC 10.33.0.0/16 (option 121 sans route du VPC en bridge)" sh -c "[ -z \"\$1\" ]" _ "$(vm_ssh vp-t33 "${ip}" 'ip -4 route show 10.33.0.0/16')" check "${ip} : pas de route vers le VPC 10.33.0.0/16 (option 121 sans route du VPC en bridge)" vm_output_empty vp-t33 "${ip}" 'ip -4 route show 10.33.0.0/16'
done done
check "t33-a1 : métadonnées de t33-a1 (instance-id)" vm_grep vp-t33 10.33.0.21 'curl -fsS http://169.254.169.254/meta-data' '^instance-id: t33-a1$' check "t33-a1 : métadonnées de t33-a1 (instance-id)" vm_grep vp-t33 10.33.0.21 'curl -fsS http://169.254.169.254/meta-data' '^instance-id: t33-a1$'
check "t33-a1 joint t33-a2, même hôte" vm_ssh vp-t33 10.33.0.21 'ping -n -c 3 -W 2 10.33.0.22' check "t33-a1 joint t33-a2, même hôte" vm_ssh vp-t33 10.33.0.21 'ping -n -c 3 -W 2 10.33.0.22'

View file

@ -58,9 +58,10 @@ done
on hv1 <<'NODE' on hv1 <<'NODE'
vm_ssh vp-t71 10.71.1.10 'pkill -x ping' || true vm_ssh vp-t71 10.71.1.10 'pkill -x ping' || true
sleep 1 sleep 1
stats=$(vm_ssh vp-t71 10.71.1.10 "awk -F'icmp_seq=' '/bytes from/ {split(\$2, a, \" \"); n++; if (a[1] > max) max = a[1]} END {print n, max}' /tmp/t71.log") stats=$(vm_ssh vp-t71 10.71.1.10 "awk -F'icmp_seq=' '/bytes from/ {split(\$2, a, \" \"); n++; if (a[1] > max) max = a[1]} END {print n+0, max+0}' /tmp/t71.log")
received=${stats% *} received=${stats% *}
sent=${stats#* } sent=${stats#* }
info "pendant tout le scénario : ${received} réponses pour ${sent} requêtes, $(( sent - received )) perdues" info "pendant tout le scénario : ${received:-0} réponses pour ${sent:-0} requêtes, $(( ${sent:-0} - ${received:-0} )) perdues"
check "aucune perte mesurable sur l'ensemble du scénario (au plus 1 %)" test $(( (sent - received) * 100 )) -le "${sent}" check "trafic pendant tout le scénario : au moins 300 requêtes émises (témoin)" test "${sent:-0}" -ge 300
check "aucune perte mesurable sur l'ensemble du scénario (au plus 1 %)" test $(( (${sent:-0} - ${received:-0}) * 100 )) -le "${sent:-0}"
NODE NODE