friedkiwi/qemu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'remotes/stefanha/tags/block-pull-request' into staging

Browse Source 
Pull request

Let's get the image fuzzer Python 3 changes merged in QEMU 4.2.

# gpg: Signature made Tue 05 Nov 2019 15:43:16 GMT
# gpg:                using RSA key 8695A8BFD3F97CDAAC35775A9CA4ABB381AB73C8
# gpg: Good signature from "Stefan Hajnoczi <stefanha@redhat.com>" [full]
# gpg:                 aka "Stefan Hajnoczi <stefanha@gmail.com>" [full]
# Primary key fingerprint: 8695 A8BF D3F9 7CDA AC35  775A 9CA4 ABB3 81AB 73C8

* remotes/stefanha/tags/block-pull-request:
  image-fuzzer: Use OSerror.strerror instead of tuple subscript
  image-fuzzer: Use errors parameter of subprocess.Popen()
  image-fuzzer: Run using python3
  image-fuzzer: Encode file name and file format to bytes
  image-fuzzer: Use bytes constant for field values
  image-fuzzer: Return bytes objects on string fuzzing functions
  image-fuzzer: Use %r for all fiels at Field.__repr__()
  image-fuzzer: Use io.StringIO
  image-fuzzer: Explicitly use integer division operator
  image-fuzzer: Write bytes instead of string to image file
  image-fuzzer: Open image files in binary mode

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell
2019-11-06 13:36:42 +00:00
parent 50d2d550d7 9fdd7860ad
commit 2dd8fd6e96
4 changed files with 63 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/image-fuzzer/qcow2/__init__.py
View File

@@ -1,2 +1 @@
from __future__ import absolute_import
from .layout import create_image

54
tests/image-fuzzer/qcow2/fuzz.py
View File

@@ -27,20 +27,20 @@
UINT32_M = 31
UINT64_M = 63
# Fuzz vectors
UINT8_V = [0, 0x10, UINT8/4, UINT8/2 - 1, UINT8/2, UINT8/2 + 1, UINT8 - 1,
UINT8_V = [0, 0x10, UINT8//4, UINT8//2 - 1, UINT8//2, UINT8//2 + 1, UINT8 - 1,
UINT8]
UINT16_V = [0, 0x100, 0x1000, UINT16/4, UINT16/2 - 1, UINT16/2, UINT16/2 + 1,
UINT16_V = [0, 0x100, 0x1000, UINT16//4, UINT16//2 - 1, UINT16//2, UINT16//2 + 1,
UINT16 - 1, UINT16]
UINT32_V = [0, 0x100, 0x1000, 0x10000, 0x100000, UINT32/4, UINT32/2 - 1,
UINT32/2, UINT32/2 + 1, UINT32 - 1, UINT32]
UINT64_V = UINT32_V + [0x1000000, 0x10000000, 0x100000000, UINT64/4,
UINT64/2 - 1, UINT64/2, UINT64/2 + 1, UINT64 - 1,
UINT32_V = [0, 0x100, 0x1000, 0x10000, 0x100000, UINT32//4, UINT32//2 - 1,
UINT32//2, UINT32//2 + 1, UINT32 - 1, UINT32]
UINT64_V = UINT32_V + [0x1000000, 0x10000000, 0x100000000, UINT64//4,
UINT64//2 - 1, UINT64//2, UINT64//2 + 1, UINT64 - 1,
UINT64]
STRING_V = ['%s%p%x%d', '.1024d', '%.2049d', '%p%p%p%p', '%x%x%x%x',
'%d%d%d%d', '%s%s%s%s', '%99999999999s', '%08x', '%%20d', '%%20n',
'%%20x', '%%20s', '%s%s%s%s%s%s%s%s%s%s', '%p%p%p%p%p%p%p%p%p%p',
'%#0123456x%08x%x%s%p%d%n%o%u%c%h%l%q%j%z%Z%t%i%e%g%f%a%C%S%08x%%',
'%s x 129', '%x x 257']
BYTES_V = [b'%s%p%x%d', b'.1024d', b'%.2049d', b'%p%p%p%p', b'%x%x%x%x',
b'%d%d%d%d', b'%s%s%s%s', b'%99999999999s', b'%08x', b'%%20d', b'%%20n',
b'%%20x', b'%%20s', b'%s%s%s%s%s%s%s%s%s%s', b'%p%p%p%p%p%p%p%p%p%p',
b'%#0123456x%08x%x%s%p%d%n%o%u%c%h%l%q%j%z%Z%t%i%e%g%f%a%C%S%08x%%',
b'%s x 129', b'%x x 257']
def random_from_intervals(intervals):
@@ -76,12 +76,12 @@ def random_bits(bit_ranges):
return val
def truncate_string(strings, length):
"""Return strings truncated to specified length."""
if type(strings) == list:
return [s[:length] for s in strings]
def truncate_bytes(sequences, length):
"""Return sequences truncated to specified length."""
if type(sequences) == list:
return [s[:length] for s in sequences]
else:
return strings[:length]
return sequences[:length]
def validator(current, pick, choices):
@@ -110,12 +110,12 @@ def bit_validator(current, bit_ranges):
return validator(current, random_bits, bit_ranges)
def string_validator(current, strings):
"""Return a random string value from the list not equal to the current.
def bytes_validator(current, sequences):
"""Return a random bytes value from the list not equal to the current.
This function is useful for selection from valid values except current one.
"""
return validator(current, random.choice, strings)
return validator(current, random.choice, sequences)
def selector(current, constraints, validate=int_validator):
@@ -283,9 +283,9 @@ def header_length(current):
def bf_name(current):
"""Fuzz the backing file name."""
constraints = [
truncate_string(STRING_V, len(current))
truncate_bytes(BYTES_V, len(current))
]
return selector(current, constraints, string_validator)
return selector(current, constraints, bytes_validator)
def ext_magic(current):
@@ -303,10 +303,10 @@ def ext_length(current):
def bf_format(current):
"""Fuzz backing file format in the corresponding header extension."""
constraints = [
truncate_string(STRING_V, len(current)),
truncate_string(STRING_V, (len(current) + 7) & ~7) # Fuzz padding
truncate_bytes(BYTES_V, len(current)),
truncate_bytes(BYTES_V, (len(current) + 7) & ~7) # Fuzz padding
]
return selector(current, constraints, string_validator)
return selector(current, constraints, bytes_validator)
def feature_type(current):
@@ -324,10 +324,10 @@ def feature_bit_number(current):
def feature_name(current):
"""Fuzz feature name field of a feature name table header extension."""
constraints = [
truncate_string(STRING_V, len(current)),
truncate_string(STRING_V, 46) # Fuzz padding (field length = 46)
truncate_bytes(BYTES_V, len(current)),
truncate_bytes(BYTES_V, 46) # Fuzz padding (field length = 46)
]
return selector(current, constraints, string_validator)
return selector(current, constraints, bytes_validator)
def l1_entry(current):

57
tests/image-fuzzer/qcow2/layout.py
View File

@@ -16,7 +16,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import absolute_import
import random
import struct
from . import fuzz
@@ -53,8 +52,8 @@ def __iter__(self):
return iter([self.fmt, self.offset, self.value, self.name])
def __repr__(self):
return "Field(fmt='%s', offset=%d, value=%s, name=%s)" % \
(self.fmt, self.offset, str(self.value), self.name)
return "Field(fmt=%r, offset=%r, value=%r, name=%r)" % \
(self.fmt, self.offset, self.value, self.name)
class FieldsList(object):
@@ -122,7 +121,7 @@ def __iter__(self):
def create_header(self, cluster_bits, backing_file_name=None):
"""Generate a random valid header."""
meta_header = [
['>4s', 0, "QFI\xfb", 'magic'],
['>4s', 0, b"QFI\xfb", 'magic'],
['>I', 4, random.randint(2, 3), 'version'],
['>Q', 8, 0, 'backing_file_offset'],
['>I', 16, 0, 'backing_file_size'],
@@ -231,7 +230,7 @@ def gen_feat_ids():
feature_tables = []
feature_ids = []
inner_offset = self.ext_offset + ext_header_len
feat_name = 'some cool feature'
feat_name = b'some cool feature'
while len(feature_tables) < num_fnt_entries * 3:
feat_type, feat_bit = gen_feat_ids()
# Remove duplicates
@@ -253,7 +252,7 @@ def gen_feat_ids():
['>I', self.ext_offset, 0x6803f857, 'ext_magic'],
# One feature table contains 3 fields and takes 48 bytes
['>I', self.ext_offset + UINT32_S,
len(feature_tables) / 3 * 48, 'ext_length']
len(feature_tables) // 3 * 48, 'ext_length']
] + feature_tables)
self.ext_offset = inner_offset
@@ -271,7 +270,7 @@ def create_l_structures(self):
def create_l2_entry(host, guest, l2_cluster):
"""Generate one L2 entry."""
offset = l2_cluster * self.cluster_size
l2_size = self.cluster_size / UINT64_S
l2_size = self.cluster_size // UINT64_S
entry_offset = offset + UINT64_S * (guest % l2_size)
cluster_descriptor = host * self.cluster_size
if not self.header['version'][0].value == 2:
@@ -283,8 +282,8 @@ def create_l2_entry(host, guest, l2_cluster):
def create_l1_entry(l2_cluster, l1_offset, guest):
"""Generate one L1 entry."""
l2_size = self.cluster_size / UINT64_S
entry_offset = l1_offset + UINT64_S * (guest / l2_size)
l2_size = self.cluster_size // UINT64_S
entry_offset = l1_offset + UINT64_S * (guest // l2_size)
# While snapshots are not supported bit #63 = 1
entry_val = (1 << 63) + l2_cluster * self.cluster_size
return ['>Q', entry_offset, entry_val, 'l1_entry']
@@ -298,11 +297,11 @@ def create_l1_entry(l2_cluster, l1_offset, guest):
l2 = []
else:
meta_data = self._get_metadata()
guest_clusters = random.sample(range(self.image_size /
guest_clusters = random.sample(range(self.image_size //
self.cluster_size),
len(self.data_clusters))
# Number of entries in a L1/L2 table
l_size = self.cluster_size / UINT64_S
l_size = self.cluster_size // UINT64_S
# Number of clusters necessary for L1 table
l1_size = int(ceil((max(guest_clusters) + 1) / float(l_size**2)))
l1_start = self._get_adjacent_clusters(self.data_clusters |
@@ -318,7 +317,7 @@ def create_l1_entry(l2_cluster, l1_offset, guest):
# L2 entries
l2 = []
for host, guest in zip(self.data_clusters, guest_clusters):
l2_id = guest / l_size
l2_id = guest // l_size
if l2_id not in l2_ids:
l2_ids.append(l2_id)
l2_clusters.append(self._get_adjacent_clusters(
@@ -339,14 +338,14 @@ def create_refcount_structures(self):
def allocate_rfc_blocks(data, size):
"""Return indices of clusters allocated for refcount blocks."""
cluster_ids = set()
diff = block_ids = set([x / size for x in data])
diff = block_ids = set([x // size for x in data])
while len(diff) != 0:
# Allocate all yet not allocated clusters
new = self._get_available_clusters(data | cluster_ids,
len(diff))
# Indices of new refcount blocks necessary to cover clusters
# in 'new'
diff = set([x / size for x in new]) - block_ids
diff = set([x // size for x in new]) - block_ids
cluster_ids |= new
block_ids |= diff
return cluster_ids, block_ids
@@ -359,7 +358,7 @@ def allocate_rfc_table(data, init_blocks, block_size):
blocks = set(init_blocks)
clusters = set()
# Number of entries in one cluster of the refcount table
size = self.cluster_size / UINT64_S
size = self.cluster_size // UINT64_S
# Number of clusters necessary for the refcount table based on
# the current number of refcount blocks
table_size = int(ceil((max(blocks) + 1) / float(size)))
@@ -373,7 +372,7 @@ def allocate_rfc_table(data, init_blocks, block_size):
table_size + 1))
# New refcount blocks necessary for clusters occupied by the
# refcount table
diff = set([c / block_size for c in table_clusters]) - blocks
diff = set([c // block_size for c in table_clusters]) - blocks
blocks |= diff
while len(diff) != 0:
# Allocate clusters for new refcount blocks
@@ -382,12 +381,12 @@ def allocate_rfc_table(data, init_blocks, block_size):
len(diff))
# Indices of new refcount blocks necessary to cover
# clusters in 'new'
diff = set([x / block_size for x in new]) - blocks
diff = set([x // block_size for x in new]) - blocks
clusters |= new
blocks |= diff
# Check if the refcount table needs one more cluster
if int(ceil((max(blocks) + 1) / float(size))) > table_size:
new_block_id = (table_start + table_size) / block_size
new_block_id = (table_start + table_size) // block_size
# Check if the additional table cluster needs
# one more refcount block
if new_block_id not in blocks:
@@ -399,13 +398,13 @@ def allocate_rfc_table(data, init_blocks, block_size):
def create_table_entry(table_offset, block_cluster, block_size,
cluster):
"""Generate a refcount table entry."""
offset = table_offset + UINT64_S * (cluster / block_size)
offset = table_offset + UINT64_S * (cluster // block_size)
return ['>Q', offset, block_cluster * self.cluster_size,
'refcount_table_entry']
def create_block_entry(block_cluster, block_size, cluster):
"""Generate a list of entries for the current block."""
entry_size = self.cluster_size / block_size
entry_size = self.cluster_size // block_size
offset = block_cluster * self.cluster_size
entry_offset = offset + entry_size * (cluster % block_size)
# While snapshots are not supported all refcounts are set to 1
@@ -415,7 +414,7 @@ def create_block_entry(block_cluster, block_size, cluster):
# Number of refcount entries per refcount block
# Convert self.cluster_size from bytes to bits to have the same
# base for the numerator and denominator
block_size = self.cluster_size * 8 / refcount_bits
block_size = self.cluster_size * 8 // refcount_bits
meta_data = self._get_metadata()
if len(self.data_clusters) == 0:
# All metadata for an empty guest image needs 4 clusters:
@@ -452,8 +451,8 @@ def create_block_entry(block_cluster, block_size, cluster):
rfc_blocks = []
for cluster in sorted(self.data_clusters | meta_data):
if cluster / block_size != block_id:
block_id = cluster / block_size
if cluster // block_size != block_id:
block_id = cluster // block_size
block_cluster = block_clusters[block_ids.index(block_id)]
rfc_table.append(create_table_entry(table_offset,
block_cluster,
@@ -503,7 +502,7 @@ def coin():
def write(self, filename):
"""Write an entire image to the file."""
image_file = open(filename, 'w')
image_file = open(filename, 'wb')
for field in self:
image_file.seek(field.offset)
image_file.write(struct.pack(field.fmt, field.value))
@@ -518,7 +517,7 @@ def write(self, filename):
rounded = (size + self.cluster_size - 1) & ~(self.cluster_size - 1)
if rounded > size:
image_file.seek(rounded - 1)
image_file.write("\0")
image_file.write(b'\x00')
image_file.close()
@staticmethod
@@ -587,7 +586,7 @@ def get_cluster_id(lst, length):
def _alloc_data(img_size, cluster_size):
"""Return a set of random indices of clusters allocated for guest data.
"""
num_of_cls = img_size/cluster_size
num_of_cls = img_size // cluster_size
return set(random.sample(range(1, num_of_cls + 1),
random.randint(0, num_of_cls)))
@@ -595,15 +594,15 @@ def _get_metadata(self):
"""Return indices of clusters allocated for image metadata."""
ids = set()
for x in self:
ids.add(x.offset/self.cluster_size)
ids.add(x.offset // self.cluster_size)
return ids
def create_image(test_img_path, backing_file_name=None, backing_file_fmt=None,
fields_to_fuzz=None):
"""Create a fuzzed image and write it to the specified file."""
image = Image(backing_file_name)
image.set_backing_file_format(backing_file_fmt)
image = Image(backing_file_name.encode())
image.set_backing_file_format(backing_file_fmt.encode())
image.create_feature_name_table()
image.set_end_of_extension_area()
image.create_l_structures()

16
tests/image-fuzzer/runner.py
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# Tool for running fuzz tests
#
@@ -18,7 +18,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import print_function
import sys
import os
import signal
@@ -28,7 +27,7 @@
from itertools import count
import time
import getopt
import StringIO
import io
import resource
try:
@@ -80,7 +79,8 @@ def handler(*args):
devnull = open('/dev/null', 'r+')
process = subprocess.Popen(q_args, stdin=devnull,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stderr=subprocess.PIPE,
errors='replace')
try:
out, err = process.communicate()
signal.alarm(0)
@@ -159,7 +159,7 @@ def __init__(self, test_id, seed, work_dir, run_log,
os.makedirs(self.current_dir)
except OSError as e:
print("Error: The working directory '%s' cannot be used. Reason: %s"\
% (self.work_dir, e[1]), file=sys.stderr)
% (self.work_dir, e.strerror), file=sys.stderr)
raise TestException
self.log = open(os.path.join(self.current_dir, "test.log"), "w")
self.parent_log = open(run_log, "a")
@@ -183,7 +183,7 @@ def _create_backing_file(self):
MAX_BACKING_FILE_SIZE) * (1 << 20)
cmd = self.qemu_img + ['create', '-f', backing_file_fmt,
backing_file_name, str(backing_file_size)]
temp_log = StringIO.StringIO()
temp_log = io.StringIO()
retcode = run_app(temp_log, cmd)
if retcode == 0:
temp_log.close()
@@ -240,13 +240,13 @@ def execute(self, input_commands=None, fuzz_config=None):
"Backing file: %s\n" \
% (self.seed, " ".join(current_cmd),
self.current_dir, backing_file_name)
temp_log = StringIO.StringIO()
temp_log = io.StringIO()
try:
retcode = run_app(temp_log, current_cmd)
except OSError as e:
multilog("%sError: Start of '%s' failed. Reason: %s\n\n"
% (test_summary, os.path.basename(current_cmd[0]),
e[1]),
e.strerror),
sys.stderr, self.log, self.parent_log)
raise TestException