From 0bf4d741ce1078f348c063ee5c6aa36ced35243f Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Thu, 11 Jul 2024 14:26:32 -0700 Subject: [PATCH 1/3] [threads] Update the fuzzer for shared types Update the fuzzer to both handle shared types in initial contents and create and use new shared types without crashing or producing invalid modules. Since V8 does not have a complete implementation of shared-everything-threads yet, disable fuzzing V8 when shared-everything is enabled. To avoid losing too much coverage of V8, disable shared-everything in the fuzzer more frequently than other features. --- scripts/fuzz_opt.py | 26 ++---- src/tools/fuzzing/fuzzing.cpp | 141 +++++++++++++++++++------------ src/tools/fuzzing/heap-types.cpp | 3 +- 3 files changed, 99 insertions(+), 71 deletions(-) diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py index f49234035c4..fe4122ef6ac 100755 --- a/scripts/fuzz_opt.py +++ b/scripts/fuzz_opt.py @@ -145,7 +145,11 @@ def randomize_feature_opts(): # 2/3 of the remaining 90% use them all. This is useful to maximize # coverage, as enabling more features enables more optimizations and # code paths, and also allows all initial contents to run. - pass + + # The shared-everything feature is new and we want to fuzz it, but it + # also currently disables fuzzing V8, so disable it half the time. + if random.random() < 0.5: + FEATURE_OPTS.append('--disable-shared-everything') print('randomized feature opts:', '\n ' + '\n '.join(FEATURE_OPTS)) @@ -350,21 +354,6 @@ def is_git_repo(): 'exception-handling.wast', 'translate-to-new-eh.wast', 'rse-eh.wast', - # Shared types implementation in progress - 'type-merging-shared.wast', - 'shared-types.wast', - 'shared-polymorphism.wast', - 'shared-struct.wast', - 'shared-array.wast', - 'shared-i31.wast', - 'shared-null.wast', - 'shared-absheaptype.wast', - 'type-ssa-shared.wast', - 'shared-ref_eq.wast', - 'shared-types-no-gc.wast', - 'shared-ref-i31.wast', - 'table-type.wast', - 'elem-type.wast', ] @@ -847,7 +836,10 @@ def run(self, wasm, extra_d8_flags=[]): return run_vm([shared.V8, FUZZ_SHELL_JS] + shared.V8_OPTS + extra_d8_flags + ['--', wasm]) def can_run(self, wasm): - return True + # V8 does not support shared memories when running with + # shared-everything enabled, so do not fuzz shared-everything + # for now. + return all_disallowed(['shared-everything']) def can_compare_to_self(self): # With nans, VM differences can confuse us, so only very simple VMs diff --git a/src/tools/fuzzing/fuzzing.cpp b/src/tools/fuzzing/fuzzing.cpp index 8e699fe1366..2aff7146e28 100644 --- a/src/tools/fuzzing/fuzzing.cpp +++ b/src/tools/fuzzing/fuzzing.cpp @@ -246,11 +246,8 @@ void TranslateToFuzzReader::setupHeapTypes() { // For GC, also generate random types. if (wasm.features.hasGC()) { - // Do not generate shared types until the fuzzer can be updated to handle - // them. - auto features = wasm.features - FeatureSet::SharedEverything; auto generator = - HeapTypeGenerator::create(random, features, upTo(MAX_NEW_GC_TYPES)); + HeapTypeGenerator::create(random, wasm.features, upTo(MAX_NEW_GC_TYPES)); auto result = generator.builder.build(); if (auto* err = result.getError()) { Fatal() << "Failed to build heap types: " << err->reason << " at index " @@ -288,10 +285,16 @@ void TranslateToFuzzReader::setupHeapTypes() { } // Basic types must be handled directly, since subTypes doesn't look at // those. + auto share = type.getShared(); + auto struct_ = HeapTypes::struct_.getBasic(share); + auto array = HeapTypes::array.getBasic(share); + auto eq = HeapTypes::eq.getBasic(share); + auto any = HeapTypes::any.getBasic(share); + auto func = HeapTypes::func.getBasic(share); if (type.isStruct()) { - interestingHeapSubTypes[HeapType::struct_].push_back(type); - interestingHeapSubTypes[HeapType::eq].push_back(type); - interestingHeapSubTypes[HeapType::any].push_back(type); + interestingHeapSubTypes[struct_].push_back(type); + interestingHeapSubTypes[eq].push_back(type); + interestingHeapSubTypes[any].push_back(type); // Note the mutable fields. auto& fields = type.getStruct().fields; @@ -301,15 +304,15 @@ void TranslateToFuzzReader::setupHeapTypes() { } } } else if (type.isArray()) { - interestingHeapSubTypes[HeapType::array].push_back(type); - interestingHeapSubTypes[HeapType::eq].push_back(type); - interestingHeapSubTypes[HeapType::any].push_back(type); + interestingHeapSubTypes[array].push_back(type); + interestingHeapSubTypes[eq].push_back(type); + interestingHeapSubTypes[any].push_back(type); if (type.getArray().element.mutable_) { mutableArrays.push_back(type); } } else if (type.isSignature()) { - interestingHeapSubTypes[HeapType::func].push_back(type); + interestingHeapSubTypes[func].push_back(type); } } @@ -2468,11 +2471,13 @@ Literal TranslateToFuzzReader::makeLiteral(Type type) { Expression* TranslateToFuzzReader::makeRefFuncConst(Type type) { auto heapType = type.getHeapType(); + auto share = heapType.getShared(); if (heapType.isBasic()) { assert(heapType.getBasic(Unshared) == HeapType::func); // With high probability, use the last created function if possible. // Otherwise, continue on to select some other function. - if (funcContext && !oneIn(4)) { + if (funcContext && funcContext->func->type.getShared() == share && + !oneIn(4)) { auto* target = funcContext->func; return builder.makeRefFunc(target->name, target->type); } @@ -2496,7 +2501,7 @@ Expression* TranslateToFuzzReader::makeRefFuncConst(Type type) { // here). if ((type.isNullable() && oneIn(2)) || (type.isNonNullable() && oneIn(16) && funcContext)) { - Expression* ret = builder.makeRefNull(HeapType::nofunc); + Expression* ret = builder.makeRefNull(HeapTypes::nofunc.getBasic(share)); if (!type.isNullable()) { assert(funcContext); ret = builder.makeRefAs(RefAsNonNull, ret); @@ -2511,7 +2516,10 @@ Expression* TranslateToFuzzReader::makeRefFuncConst(Type type) { if (heapType.isBasic()) { // We need a specific signature type to create a function. Pick an arbitrary // signature if we only had generic 'func' here. - heapType = Signature(Type::none, Type::none); + TypeBuilder builder(1); + builder[0] = Signature(Type::none, Type::none); + builder[0].setShared(share); + heapType = (*builder.build())[0]; } auto* body = heapType.getSignature().results == Type::none ? (Expression*)builder.makeNop() @@ -2553,10 +2561,10 @@ Expression* TranslateToFuzzReader::makeBasicRef(Type type) { auto heapType = type.getHeapType(); assert(heapType.isBasic()); assert(wasm.features.hasReferenceTypes()); - assert(!heapType.isShared() && "TODO: handle shared types"); + auto share = heapType.getShared(); switch (heapType.getBasic(Unshared)) { case HeapType::ext: { - auto null = builder.makeRefNull(HeapType::ext); + auto null = builder.makeRefNull(HeapTypes::ext.getBasic(share)); // TODO: support actual non-nullable externrefs via imported globals or // similar. if (!type.isNullable()) { @@ -2575,12 +2583,16 @@ Expression* TranslateToFuzzReader::makeBasicRef(Type type) { // Choose a subtype we can materialize a constant for. We cannot // materialize non-nullable refs to func or i31 in global contexts. Nullability nullability = getSubType(type.getNullability()); - auto subtype = pick(FeatureOptions() - .add(FeatureSet::ReferenceTypes | FeatureSet::GC, - HeapType::i31, - HeapType::struct_, - HeapType::array) - .add(FeatureSet::Strings, HeapType::string)); + auto subtypeOpts = FeatureOptions().add( + FeatureSet::ReferenceTypes | FeatureSet::GC, + HeapType::i31, + HeapType::struct_, + HeapType::array); + if (share == Unshared) { + // Shared strings not yet supported. + subtypeOpts.add(FeatureSet::Strings, HeapType::string); + } + auto subtype = pick(subtypeOpts).getBasic(share); return makeConst(Type(subtype, nullability)); } case HeapType::eq: { @@ -2589,7 +2601,7 @@ Expression* TranslateToFuzzReader::makeBasicRef(Type type) { // a subtype of anyref, but we cannot create constants of it, except // for null. assert(type.isNullable()); - return builder.makeRefNull(HeapType::none); + return builder.makeRefNull(HeapTypes::none.getBasic(share)); } auto nullability = getSubType(type.getNullability()); // ref.i31 is not allowed in initializer expressions. @@ -2605,14 +2617,14 @@ Expression* TranslateToFuzzReader::makeBasicRef(Type type) { subtype = HeapType::array; break; } - return makeConst(Type(subtype, nullability)); + return makeConst(Type(subtype.getBasic(share), nullability)); } case HeapType::i31: { assert(wasm.features.hasGC()); if (type.isNullable() && oneIn(4)) { - return builder.makeRefNull(HeapType::none); + return builder.makeRefNull(HeapTypes::none.getBasic(share)); } - return builder.makeRefI31(makeConst(Type::i32)); + return builder.makeRefI31(makeConst(Type::i32), share); } case HeapType::struct_: { assert(wasm.features.hasGC()); @@ -2621,15 +2633,29 @@ Expression* TranslateToFuzzReader::makeBasicRef(Type type) { // Use a local static to avoid the expense of canonicalizing a new type // every time. static HeapType trivialStruct = HeapType(Struct()); - return builder.makeStructNew(trivialStruct, std::vector{}); + static HeapType sharedTrivialStruct = []() { + TypeBuilder builder(1); + builder[0] = Struct{}; + builder[0].setShared(); + return (*builder.build())[0]; + }(); + auto ht = share == Shared ? sharedTrivialStruct : trivialStruct; + return builder.makeStructNew(ht, std::vector{}); } case HeapType::array: { static HeapType trivialArray = HeapType(Array(Field(Field::PackedType::i8, Immutable))); - return builder.makeArrayNewFixed(trivialArray, {}); + static HeapType sharedTrivialArray = []() { + TypeBuilder builder(1); + builder[0] = Array(Field(Field::PackedType::i8, Immutable)); + builder[0].setShared(); + return (*builder.build())[0]; + }(); + auto ht = share == Shared ? sharedTrivialArray : trivialArray; + return builder.makeArrayNewFixed(ht, {}); } case HeapType::exn: { - auto null = builder.makeRefNull(HeapType::exn); + auto null = builder.makeRefNull(HeapTypes::exn.getBasic(share)); if (!type.isNullable()) { assert(funcContext); return builder.makeRefAs(RefAsNonNull, null); @@ -2638,6 +2664,7 @@ Expression* TranslateToFuzzReader::makeBasicRef(Type type) { } case HeapType::string: { // In non-function contexts all we can do is string.const. + assert(share == Unshared && "shared strings not supported"); if (!funcContext) { return makeStringConst(); } @@ -2671,7 +2698,7 @@ Expression* TranslateToFuzzReader::makeBasicRef(Type type) { case HeapType::nofunc: case HeapType::nocont: case HeapType::noexn: { - auto null = builder.makeRefNull(heapType); + auto null = builder.makeRefNull(heapType.getBasic(share)); if (!type.isNullable()) { assert(funcContext); return builder.makeRefAs(RefAsNonNull, null); @@ -4231,48 +4258,56 @@ HeapType TranslateToFuzzReader::getSubType(HeapType type) { return type; } if (type.isBasic() && oneIn(2)) { - assert(!type.isShared() && "TODO: handle shared types"); + auto share = type.getShared(); switch (type.getBasic(Unshared)) { case HeapType::func: // TODO: Typed function references. return pick(FeatureOptions() .add(FeatureSet::ReferenceTypes, HeapType::func) - .add(FeatureSet::GC, HeapType::nofunc)); + .add(FeatureSet::GC, HeapType::nofunc)) + .getBasic(share); case HeapType::cont: - return pick(HeapType::cont, HeapType::nocont); + return pick(HeapTypes::cont, HeapTypes::nocont).getBasic(share); case HeapType::ext: return pick(FeatureOptions() .add(FeatureSet::ReferenceTypes, HeapType::ext) - .add(FeatureSet::GC, HeapType::noext)); - case HeapType::any: + .add(FeatureSet::GC, HeapType::noext)) + .getBasic(share); + case HeapType::any: { assert(wasm.features.hasReferenceTypes()); assert(wasm.features.hasGC()); - return pick(FeatureOptions() - .add(FeatureSet::GC, - HeapType::any, - HeapType::eq, - HeapType::i31, - HeapType::struct_, - HeapType::array, - HeapType::none) - .add(FeatureSet::Strings, HeapType::string)); + auto options = FeatureOptions().add(FeatureSet::GC, + HeapType::any, + HeapType::eq, + HeapType::i31, + HeapType::struct_, + HeapType::array, + HeapType::none); + if (share == Unshared) { + // Shared strings not yet supported. + options.add(FeatureSet::Strings, HeapType::string); + } + return pick(options).getBasic(share); + } case HeapType::eq: assert(wasm.features.hasReferenceTypes()); assert(wasm.features.hasGC()); - return pick(HeapType::eq, - HeapType::i31, - HeapType::struct_, - HeapType::array, - HeapType::none); + return pick(HeapTypes::eq, + HeapTypes::i31, + HeapTypes::struct_, + HeapTypes::array, + HeapTypes::none) + .getBasic(share); case HeapType::i31: - return pick(HeapType::i31, HeapType::none); + return pick(HeapTypes::i31, HeapTypes::none).getBasic(share); case HeapType::struct_: - return pick(HeapType::struct_, HeapType::none); + return pick(HeapTypes::struct_, HeapTypes::none).getBasic(share); case HeapType::array: - return pick(HeapType::array, HeapType::none); + return pick(HeapTypes::array, HeapTypes::none).getBasic(share); case HeapType::exn: - return HeapType::exn; + return HeapTypes::exn.getBasic(share); case HeapType::string: + assert(share == Unshared); return HeapType::string; case HeapType::none: case HeapType::noext: diff --git a/src/tools/fuzzing/heap-types.cpp b/src/tools/fuzzing/heap-types.cpp index a1c879b4286..c1c13bc0a02 100644 --- a/src/tools/fuzzing/heap-types.cpp +++ b/src/tools/fuzzing/heap-types.cpp @@ -759,7 +759,8 @@ void Inhabitator::markExternRefsNullable() { auto children = type.getTypeChildren(); for (size_t i = 0; i < children.size(); ++i) { auto child = children[i]; - if (child.isRef() && child.getHeapType() == HeapType::ext && + if (child.isRef() && child.getHeapType().isBasic() && + child.getHeapType().getBasic(Unshared) == HeapType::ext && child.isNonNullable()) { markNullable({type, i}); } From 2ea1d3e972df716c919d80ee3d040d4b44c0e9f9 Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Thu, 18 Jul 2024 09:18:52 -0700 Subject: [PATCH 2/3] increase frequency of disabling shared-everything --- scripts/fuzz_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py index fe4122ef6ac..77fc664c94a 100755 --- a/scripts/fuzz_opt.py +++ b/scripts/fuzz_opt.py @@ -147,8 +147,8 @@ def randomize_feature_opts(): # code paths, and also allows all initial contents to run. # The shared-everything feature is new and we want to fuzz it, but it - # also currently disables fuzzing V8, so disable it half the time. - if random.random() < 0.5: + # also currently disables fuzzing V8, so disable it most of the time. + if random.random() < 0.9: FEATURE_OPTS.append('--disable-shared-everything') print('randomized feature opts:', '\n ' + '\n '.join(FEATURE_OPTS)) From ec3bc3ee74c461c97f4ef0943dbafdecd19b617b Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Thu, 18 Jul 2024 09:24:31 -0700 Subject: [PATCH 3/3] update test --- ...e-to-fuzz_all-features_metrics_noprint.txt | 83 +++++++++---------- 1 file changed, 37 insertions(+), 46 deletions(-) diff --git a/test/passes/translate-to-fuzz_all-features_metrics_noprint.txt b/test/passes/translate-to-fuzz_all-features_metrics_noprint.txt index aba60b9da76..6db0f908dd4 100644 --- a/test/passes/translate-to-fuzz_all-features_metrics_noprint.txt +++ b/test/passes/translate-to-fuzz_all-features_metrics_noprint.txt @@ -1,61 +1,52 @@ total - [exports] : 3 - [funcs] : 4 - [globals] : 24 + [exports] : 5 + [funcs] : 9 + [globals] : 26 [imports] : 5 [memories] : 1 [memory-data] : 20 [table-data] : 3 [tables] : 1 - [tags] : 1 - [total] : 846 - [vars] : 38 - ArrayCopy : 1 - ArrayGet : 3 - ArrayLen : 5 - ArrayNew : 24 - ArrayNewFixed : 1 - ArraySet : 1 + [tags] : 2 + [total] : 669 + [vars] : 27 + ArrayNew : 16 + ArrayNewFixed : 3 AtomicCmpxchg : 1 AtomicFence : 1 - AtomicNotify : 1 - AtomicRMW : 1 - Binary : 91 - Block : 75 - Break : 17 - Call : 13 - Const : 177 + Binary : 75 + Block : 70 + Break : 7 + Call : 26 + CallRef : 1 + Const : 143 Drop : 3 - GlobalGet : 50 - GlobalSet : 26 - I31Get : 2 - If : 26 - Load : 23 - LocalGet : 79 - LocalSet : 56 - Loop : 10 - MemoryCopy : 1 - Nop : 13 - Pop : 4 - RefAs : 16 - RefEq : 1 + GlobalGet : 37 + GlobalSet : 27 + I31Get : 1 + If : 20 + Load : 21 + LocalGet : 55 + LocalSet : 40 + Loop : 6 + Nop : 5 + Pop : 5 + RefAs : 2 + RefEq : 2 RefFunc : 5 - RefI31 : 5 - RefIsNull : 2 - RefNull : 23 - RefTest : 3 - Return : 2 - SIMDTernary : 1 - Select : 4 - Store : 2 + RefI31 : 2 + RefNull : 11 + RefTest : 2 + Return : 6 + Select : 2 StringConst : 6 - StringEncode : 1 + StringEq : 1 StringMeasure : 1 StringWTF16Get : 1 - StructGet : 1 - StructNew : 14 + StructNew : 17 StructSet : 1 Try : 4 - TupleMake : 6 - Unary : 29 - Unreachable : 13 + TupleExtract : 3 + TupleMake : 5 + Unary : 20 + Unreachable : 15