nbd patches for 2022-03-07

- Dan Berrange: Allow qemu-nbd to support TLS over Unix sockets
 - Eric Blake: Minor cleanups related to 64-bit block operations
 -----BEGIN PGP SIGNATURE-----
 
 iQEzBAABCAAdFiEEccLMIrHEYCkn0vOqp6FrSiUnQ2oFAmImtE8ACgkQp6FrSiUn
 Q2ovmgf/aksDqf2eNcahs++fez+8Qi9ll5OY/qGyjnzBgsatYKjrK+xF7OnjoJox
 eRX026lh81Q4EQK7oZBUnr2UCY4bncDBTI7MTLh603EV/tId5ZLwx007ERhzvtC1
 mIsQHXNuO9X25LQG2eWnfunY9YztQpiT5r/g3khD2yPBqJWIvBfblzPLx6FkF7px
 /WM8xEKCihmGr1Wr3b+zGYL083YkaBWCvHoR8mJt3tEFUj+Qie8XcdV0OVyI0XUj
 5goIFRcpVwBE8P2nLtfUKNzEXz22cmdonOJUX7E5IvGO21k5F/HrWlHdo8JnuSUZ
 t0w5L9yCxBrRpY1burz30b77J0WMCw==
 =C8Dd
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/ericb/tags/pull-nbd-2022-03-07' into staging

nbd patches for 2022-03-07

- Dan Berrange: Allow qemu-nbd to support TLS over Unix sockets
- Eric Blake: Minor cleanups related to 64-bit block operations

# gpg: Signature made Tue 08 Mar 2022 01:41:35 GMT
# gpg:                using RSA key 71C2CC22B1C4602927D2F3AAA7A16B4A2527436A
# gpg: Good signature from "Eric Blake <eblake@redhat.com>" [full]
# gpg:                 aka "Eric Blake (Free Software Programmer) <ebb9@byu.net>" [full]
# gpg:                 aka "[jpeg image of size 6874]" [full]
# Primary key fingerprint: 71C2 CC22 B1C4 6029 27D2  F3AA A7A1 6B4A 2527 436A

* remotes/ericb/tags/pull-nbd-2022-03-07:
  qemu-io: Allow larger write zeroes under no fallback
  qemu-io: Utilize 64-bit status during map
  nbd/server: Minor cleanups
  tests/qemu-iotests: validate NBD TLS with UNIX sockets and PSK
  tests/qemu-iotests: validate NBD TLS with UNIX sockets
  tests/qemu-iotests: validate NBD TLS with hostname mismatch
  tests/qemu-iotests: convert NBD TLS test to use standard filters
  tests/qemu-iotests: introduce filter for qemu-nbd export list
  tests/qemu-iotests: expand _filter_nbd rules
  tests/qemu-iotests: add QEMU_IOTESTS_REGEN=1 to update reference file
  block/nbd: don't restrict TLS usage to IP sockets
  qemu-nbd: add --tls-hostname option for TLS certificate validation
  block/nbd: support override of hostname for TLS certificate validation
  block: pass desired TLS hostname through from block driver client
  crypto: mandate a hostname when checking x509 creds on a client

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell 2022-03-09 11:38:29 +00:00
commit fdee2c9692
17 changed files with 268 additions and 69 deletions

View file

@ -61,11 +61,13 @@ tls_x509_create_server "ca1" "server1"
tls_x509_create_client "ca1" "client1"
tls_x509_create_client "ca2" "client2"
tls_x509_create_client "ca1" "client3"
tls_psk_create_creds "psk1"
tls_psk_create_creds "psk2"
echo
echo "== preparing image =="
_make_test_img 64M
$QEMU_IO -c 'w -P 0x11 1m 1m' "$TEST_IMG" | _filter_qemu_io
$QEMU_IO -c 'w -P 0x11 1m 1m' "$TEST_IMG" 2>&1 | _filter_qemu_io
echo
echo "== check TLS client to plain server fails =="
@ -74,9 +76,9 @@ nbd_server_start_tcp_socket -f $IMGFMT "$TEST_IMG" 2> "$TEST_DIR/server.log"
obj=tls-creds-x509,dir=${tls_dir}/client1,endpoint=client,id=tls0
$QEMU_IMG info --image-opts --object $obj \
driver=nbd,host=$nbd_tcp_addr,port=$nbd_tcp_port,tls-creds=tls0 \
2>&1 | sed "s/$nbd_tcp_port/PORT/g"
2>&1 | _filter_nbd
$QEMU_NBD_PROG -L -b $nbd_tcp_addr -p $nbd_tcp_port --object $obj \
--tls-creds=tls0
--tls-creds=tls0 2>&1 | _filter_qemu_nbd_exports
nbd_server_stop
@ -88,8 +90,10 @@ nbd_server_start_tcp_socket \
--tls-creds tls0 \
-f $IMGFMT "$TEST_IMG" 2>> "$TEST_DIR/server.log"
$QEMU_IMG info nbd://localhost:$nbd_tcp_port 2>&1 | sed "s/$nbd_tcp_port/PORT/g"
$QEMU_NBD_PROG -L -b $nbd_tcp_addr -p $nbd_tcp_port
$QEMU_IMG info nbd://localhost:$nbd_tcp_port \
2>&1 | _filter_nbd
$QEMU_NBD_PROG -L -b $nbd_tcp_addr -p $nbd_tcp_port \
2>&1 | _filter_qemu_nbd_exports
echo
echo "== check TLS works =="
@ -97,21 +101,39 @@ obj1=tls-creds-x509,dir=${tls_dir}/client1,endpoint=client,id=tls0
obj2=tls-creds-x509,dir=${tls_dir}/client3,endpoint=client,id=tls0
$QEMU_IMG info --image-opts --object $obj1 \
driver=nbd,host=$nbd_tcp_addr,port=$nbd_tcp_port,tls-creds=tls0 \
2>&1 | sed "s/$nbd_tcp_port/PORT/g"
2>&1 | _filter_nbd
$QEMU_IMG info --image-opts --object $obj2 \
driver=nbd,host=$nbd_tcp_addr,port=$nbd_tcp_port,tls-creds=tls0 \
2>&1 | sed "s/$nbd_tcp_port/PORT/g"
2>&1 | _filter_nbd
$QEMU_NBD_PROG -L -b $nbd_tcp_addr -p $nbd_tcp_port --object $obj1 \
--tls-creds=tls0
--tls-creds=tls0 2>&1 | _filter_qemu_nbd_exports
echo
echo "== check TLS fail over TCP with mismatched hostname =="
obj1=tls-creds-x509,dir=${tls_dir}/client1,endpoint=client,id=tls0
$QEMU_IMG info --image-opts --object $obj1 \
driver=nbd,host=localhost,port=$nbd_tcp_port,tls-creds=tls0 \
2>&1 | _filter_nbd
$QEMU_NBD_PROG -L -b localhost -p $nbd_tcp_port --object $obj1 \
--tls-creds=tls0 | _filter_qemu_nbd_exports
echo
echo "== check TLS works over TCP with mismatched hostname and override =="
obj1=tls-creds-x509,dir=${tls_dir}/client1,endpoint=client,id=tls0
$QEMU_IMG info --image-opts --object $obj1 \
driver=nbd,host=localhost,port=$nbd_tcp_port,tls-creds=tls0,tls-hostname=127.0.0.1 \
2>&1 | _filter_nbd
$QEMU_NBD_PROG -L -b localhost -p $nbd_tcp_port --object $obj1 \
--tls-creds=tls0 --tls-hostname=127.0.0.1 | _filter_qemu_nbd_exports
echo
echo "== check TLS with different CA fails =="
obj=tls-creds-x509,dir=${tls_dir}/client2,endpoint=client,id=tls0
$QEMU_IMG info --image-opts --object $obj \
driver=nbd,host=$nbd_tcp_addr,port=$nbd_tcp_port,tls-creds=tls0 \
2>&1 | sed "s/$nbd_tcp_port/PORT/g"
2>&1 | _filter_nbd
$QEMU_NBD_PROG -L -b $nbd_tcp_addr -p $nbd_tcp_port --object $obj \
--tls-creds=tls0
--tls-creds=tls0 2>&1 | _filter_qemu_nbd_exports
echo
echo "== perform I/O over TLS =="
@ -121,7 +143,8 @@ $QEMU_IO -c 'r -P 0x11 1m 1m' -c 'w -P 0x22 1m 1m' --image-opts \
driver=nbd,host=$nbd_tcp_addr,port=$nbd_tcp_port,tls-creds=tls0 \
2>&1 | _filter_qemu_io
$QEMU_IO -f $IMGFMT -r -U -c 'r -P 0x22 1m 1m' "$TEST_IMG" | _filter_qemu_io
$QEMU_IO -f $IMGFMT -r -U -c 'r -P 0x22 1m 1m' "$TEST_IMG" \
2>&1 | _filter_qemu_io
echo
echo "== check TLS with authorization =="
@ -139,12 +162,62 @@ nbd_server_start_tcp_socket \
$QEMU_IMG info --image-opts \
--object tls-creds-x509,dir=${tls_dir}/client1,endpoint=client,id=tls0 \
driver=nbd,host=$nbd_tcp_addr,port=$nbd_tcp_port,tls-creds=tls0 \
2>&1 | sed "s/$nbd_tcp_port/PORT/g"
2>&1 | _filter_nbd
$QEMU_IMG info --image-opts \
--object tls-creds-x509,dir=${tls_dir}/client3,endpoint=client,id=tls0 \
driver=nbd,host=$nbd_tcp_addr,port=$nbd_tcp_port,tls-creds=tls0 \
2>&1 | sed "s/$nbd_tcp_port/PORT/g"
2>&1 | _filter_nbd
nbd_server_stop
nbd_server_start_unix_socket \
--object tls-creds-x509,dir=${tls_dir}/server1,endpoint=server,id=tls0,verify-peer=on \
--tls-creds tls0 \
-f $IMGFMT "$TEST_IMG" 2>> "$TEST_DIR/server.log"
echo
echo "== check TLS fail over UNIX with no hostname =="
obj1=tls-creds-x509,dir=${tls_dir}/client1,endpoint=client,id=tls0
$QEMU_IMG info --image-opts --object $obj1 \
driver=nbd,path=$nbd_unix_socket,tls-creds=tls0 2>&1 | _filter_nbd
$QEMU_NBD_PROG -L -k $nbd_unix_socket --object $obj1 --tls-creds=tls0 \
2>&1 | _filter_qemu_nbd_exports
echo
echo "== check TLS works over UNIX with hostname override =="
obj1=tls-creds-x509,dir=${tls_dir}/client1,endpoint=client,id=tls0
$QEMU_IMG info --image-opts --object $obj1 \
driver=nbd,path=$nbd_unix_socket,tls-creds=tls0,tls-hostname=127.0.0.1 \
2>&1 | _filter_nbd
$QEMU_NBD_PROG -L -k $nbd_unix_socket --object $obj1 \
--tls-creds=tls0 --tls-hostname=127.0.0.1 2>&1 | _filter_qemu_nbd_exports
echo
echo "== check TLS works over UNIX with PSK =="
nbd_server_stop
nbd_server_start_unix_socket \
--object tls-creds-psk,dir=${tls_dir}/psk1,endpoint=server,id=tls0,verify-peer=on \
--tls-creds tls0 \
-f $IMGFMT "$TEST_IMG" 2>> "$TEST_DIR/server.log"
obj1=tls-creds-psk,dir=${tls_dir}/psk1,username=psk1,endpoint=client,id=tls0
$QEMU_IMG info --image-opts --object $obj1 \
driver=nbd,path=$nbd_unix_socket,tls-creds=tls0 \
2>&1 | _filter_nbd
$QEMU_NBD_PROG -L -k $nbd_unix_socket --object $obj1 \
--tls-creds=tls0 2>&1 | _filter_qemu_nbd_exports
echo
echo "== check TLS fails over UNIX with mismatch PSK =="
obj1=tls-creds-psk,dir=${tls_dir}/psk2,username=psk2,endpoint=client,id=tls0
$QEMU_IMG info --image-opts --object $obj1 \
driver=nbd,path=$nbd_unix_socket,tls-creds=tls0 \
2>&1 | _filter_nbd
$QEMU_NBD_PROG -L -k $nbd_unix_socket --object $obj1 \
--tls-creds=tls0 2>&1 | _filter_qemu_nbd_exports
echo
echo "== final server log =="

View file

@ -7,6 +7,8 @@ Generating a signed certificate...
Generating a signed certificate...
Generating a signed certificate...
Generating a signed certificate...
Generating a random key for user 'psk1'
Generating a random key for user 'psk2'
== preparing image ==
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
@ -17,15 +19,12 @@ wrote 1048576/1048576 bytes at offset 1048576
qemu-img: Could not open 'driver=nbd,host=127.0.0.1,port=PORT,tls-creds=tls0': Denied by server for option 5 (starttls)
server reported: TLS not configured
qemu-nbd: Denied by server for option 5 (starttls)
server reported: TLS not configured
== check plain client to TLS server fails ==
qemu-img: Could not open 'nbd://localhost:PORT': TLS negotiation required before option 7 (go)
Did you forget a valid tls-creds?
server reported: Option 0x7 not permitted before TLS
qemu-nbd: TLS negotiation required before option 3 (list)
Did you forget a valid tls-creds?
server reported: Option 0x3 not permitted before TLS
== check TLS works ==
image: nbd://127.0.0.1:PORT
@ -39,12 +38,21 @@ disk size: unavailable
exports available: 1
export: ''
size: 67108864
flags: 0xced ( flush fua trim zeroes df cache fast-zero )
min block: 1
opt block: 4096
max block: 33554432
available meta contexts: 1
base:allocation
== check TLS fail over TCP with mismatched hostname ==
qemu-img: Could not open 'driver=nbd,host=localhost,port=PORT,tls-creds=tls0': Certificate does not match the hostname localhost
qemu-nbd: Certificate does not match the hostname localhost
== check TLS works over TCP with mismatched hostname and override ==
image: nbd://localhost:PORT
file format: nbd
virtual size: 64 MiB (67108864 bytes)
disk size: unavailable
exports available: 1
export: ''
size: 67108864
min block: 1
== check TLS with different CA fails ==
qemu-img: Could not open 'driver=nbd,host=127.0.0.1,port=PORT,tls-creds=tls0': The certificate hasn't got a known issuer
@ -62,9 +70,43 @@ read 1048576/1048576 bytes at offset 1048576
qemu-img: Could not open 'driver=nbd,host=127.0.0.1,port=PORT,tls-creds=tls0': Failed to read option reply: Cannot read from TLS channel: Software caused connection abort
qemu-img: Could not open 'driver=nbd,host=127.0.0.1,port=PORT,tls-creds=tls0': Failed to read option reply: Cannot read from TLS channel: Software caused connection abort
== check TLS fail over UNIX with no hostname ==
qemu-img: Could not open 'driver=nbd,path=SOCK_DIR/qemu-nbd.sock,tls-creds=tls0': No hostname for certificate validation
qemu-nbd: No hostname for certificate validation
== check TLS works over UNIX with hostname override ==
image: nbd+unix://?socket=SOCK_DIR/qemu-nbd.sock
file format: nbd
virtual size: 64 MiB (67108864 bytes)
disk size: unavailable
exports available: 1
export: ''
size: 67108864
min block: 1
== check TLS works over UNIX with PSK ==
image: nbd+unix://?socket=SOCK_DIR/qemu-nbd.sock
file format: nbd
virtual size: 64 MiB (67108864 bytes)
disk size: unavailable
exports available: 1
export: ''
size: 67108864
min block: 1
== check TLS fails over UNIX with mismatch PSK ==
qemu-img: Could not open 'driver=nbd,path=SOCK_DIR/qemu-nbd.sock,tls-creds=tls0': TLS handshake failed: The TLS connection was non-properly terminated.
qemu-nbd: TLS handshake failed: The TLS connection was non-properly terminated.
== final server log ==
qemu-nbd: option negotiation failed: Failed to read opts magic: Cannot read from TLS channel: Software caused connection abort
qemu-nbd: option negotiation failed: Failed to read opts magic: Cannot read from TLS channel: Software caused connection abort
qemu-nbd: option negotiation failed: Verify failed: No certificate was found.
qemu-nbd: option negotiation failed: Verify failed: No certificate was found.
qemu-nbd: option negotiation failed: TLS x509 authz check for DISTINGUISHED-NAME is denied
qemu-nbd: option negotiation failed: TLS x509 authz check for DISTINGUISHED-NAME is denied
qemu-nbd: option negotiation failed: Failed to read opts magic: Cannot read from TLS channel: Software caused connection abort
qemu-nbd: option negotiation failed: Failed to read opts magic: Cannot read from TLS channel: Software caused connection abort
qemu-nbd: option negotiation failed: TLS handshake failed: An illegal parameter has been received.
qemu-nbd: option negotiation failed: TLS handshake failed: An illegal parameter has been received.
*** done

View file

@ -58,7 +58,7 @@ echo
nbd_server_start_unix_socket -f $IMGFMT "$TEST_IMG_FILE"
$QEMU_NBD_PROG --list -k $nbd_unix_socket | grep '\(size\|min\)'
$QEMU_NBD_PROG --list -k $nbd_unix_socket | _filter_qemu_nbd_exports
$QEMU_IMG map -f raw --output=json "$TEST_IMG" | _filter_qemu_img_map
$QEMU_IO -f raw -c map "$TEST_IMG"
nbd_server_stop
@ -71,7 +71,7 @@ echo
# sector alignment, here at the server.
nbd_server_start_unix_socket "$TEST_IMG_FILE" 2> "$TEST_DIR/server.log"
$QEMU_NBD_PROG --list -k $nbd_unix_socket | grep '\(size\|min\)'
$QEMU_NBD_PROG --list -k $nbd_unix_socket | _filter_qemu_nbd_exports
$QEMU_IMG map -f raw --output=json "$TEST_IMG" | _filter_qemu_img_map
$QEMU_IO -f raw -c map "$TEST_IMG"
nbd_server_stop
@ -84,7 +84,7 @@ echo
# Now force sector alignment at the client.
nbd_server_start_unix_socket -f $IMGFMT "$TEST_IMG_FILE"
$QEMU_NBD_PROG --list -k $nbd_unix_socket | grep '\(size\|min\)'
$QEMU_NBD_PROG --list -k $nbd_unix_socket | _filter_qemu_nbd_exports
$QEMU_IMG map --output=json "$TEST_IMG" | _filter_qemu_img_map
$QEMU_IO -c map "$TEST_IMG"
nbd_server_stop

View file

@ -2,6 +2,8 @@ QA output created by 241
=== Exporting unaligned raw image, natural alignment ===
exports available: 1
export: ''
size: 1024
min block: 1
[{ "start": 0, "length": 1000, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET},
@ -10,6 +12,8 @@ QA output created by 241
=== Exporting unaligned raw image, forced server sector alignment ===
exports available: 1
export: ''
size: 1024
min block: 512
[{ "start": 0, "length": 1024, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET}]
@ -20,6 +24,8 @@ WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed
=== Exporting unaligned raw image, forced client sector alignment ===
exports available: 1
export: ''
size: 1024
min block: 1
[{ "start": 0, "length": 1000, "depth": 0, "present": true, "zero": false, "data": true, "offset": OFFSET},

View file

@ -301,10 +301,19 @@ _filter_nbd()
# Filter out the TCP port number since this changes between runs.
sed -e '/nbd\/.*\.c:/d' \
-e 's#127\.0\.0\.1:[0-9]*#127.0.0.1:PORT#g' \
-e 's#localhost:[0-9]*#localhost:PORT#g' \
-e 's#host=127\.0\.0\.1,port=[0-9]*#host=127.0.0.1,port=PORT#g' \
-e 's#host=localhost,port=[0-9]*#host=localhost,port=PORT#g' \
-e "s#path=$SOCK_DIR#path=SOCK_DIR#g" \
-e "s#?socket=$SOCK_DIR#?socket=SOCK_DIR#g" \
-e 's#\(foo\|PORT/\?\|.sock\): Failed to .*$#\1#'
}
_filter_qemu_nbd_exports()
{
grep '\(exports available\|export\|size\|min block\|qemu-nbd\):'
}
_filter_qmp_empty_return()
{
grep -v '{"return": {}}'

View file

@ -24,6 +24,7 @@ tls_x509_cleanup()
{
rm -f "${tls_dir}"/*.pem
rm -f "${tls_dir}"/*/*.pem
rm -f "${tls_dir}"/*/*.psk
rmdir "${tls_dir}"/*
rmdir "${tls_dir}"
}
@ -40,6 +41,18 @@ tls_certtool()
rm -f "${tls_dir}"/certtool.log
}
tls_psktool()
{
psktool "$@" 1>"${tls_dir}"/psktool.log 2>&1
if test "$?" = 0; then
head -1 "${tls_dir}"/psktool.log
else
cat "${tls_dir}"/psktool.log
fi
rm -f "${tls_dir}"/psktool.log
}
tls_x509_init()
{
(certtool --help) >/dev/null 2>&1 || \
@ -118,12 +131,13 @@ tls_x509_create_server()
caname=$1
name=$2
# We don't include 'localhost' in the cert, as
# we want to keep it unlisted to let tests
# validate hostname override
mkdir -p "${tls_dir}/$name"
cat > "${tls_dir}/cert.info" <<EOF
organization = Cthulhu Dark Lord Enterprises $name
cn = localhost
dns_name = localhost
dns_name = localhost.localdomain
cn = iotests.qemu.org
ip_address = 127.0.0.1
ip_address = ::1
tls_www_server
@ -175,3 +189,14 @@ EOF
rm -f "${tls_dir}/cert.info"
}
tls_psk_create_creds()
{
name=$1
mkdir -p "${tls_dir}/$name"
tls_psktool \
--pskfile "${tls_dir}/$name/keys.psk" \
--username "$name"
}

View file

@ -25,6 +25,7 @@ import subprocess
import contextlib
import json
import termios
import shutil
import sys
from multiprocessing import Pool
from contextlib import contextmanager
@ -322,6 +323,11 @@ class TestRunner(ContextManager['TestRunner']):
diff = file_diff(str(f_reference), str(f_bad))
if diff:
if os.environ.get("QEMU_IOTESTS_REGEN", None) is not None:
shutil.copyfile(str(f_bad), str(f_reference))
print("########################################")
print("##### REFERENCE FILE UPDATED #####")
print("########################################")
return TestResult(status='fail', elapsed=elapsed,
description=f'output mismatch (see {f_bad})',
diff=diff, casenotrun=casenotrun)