diff --git a/src/Expr.cpp b/src/Expr.cpp index 7d55fe9350c4..84093736d65e 100644 --- a/src/Expr.cpp +++ b/src/Expr.cpp @@ -1,3 +1,5 @@ +#include + #include "Expr.h" #include "IROperator.h" // for lossless_cast() @@ -35,6 +37,12 @@ const IntImm *IntImm::make(Type t, int64_t value) { IntImm *node = new IntImm; node->type = t; node->value = value; + // Small values are extremely common, so a hash that just slices up the + // bits of the value (like combine_hash below) would put all the entropy + // for those in the low bits, which get discarded by set_hash. Multiply + // by a large odd constant and keep the high bits instead, which mixes + // in the low bits of the value even when the value itself is small. + node->set_hash((uint32_t)((((uint64_t)value) * 0x9e3779b97f4a7c15ULL) >> 32)); return node; } @@ -51,6 +59,9 @@ const UIntImm *UIntImm::make(Type t, uint64_t value) { UIntImm *node = new UIntImm; node->type = t; node->value = value; + // See the comment in IntImm::make about why we multiply rather than + // just slicing up the bits of the value. + node->set_hash((uint32_t)((value * 0x9e3779b97f4a7c15ULL) >> 32)); return node; } @@ -77,6 +88,7 @@ const FloatImm *FloatImm::make(Type t, double value) { internal_error << "FloatImm must be 16, 32, or 64-bit\n"; } + node->set_hash((uint32_t)std::hash{}(node->value)); return node; } @@ -84,6 +96,7 @@ const StringImm *StringImm::make(const std::string &val) { StringImm *node = new StringImm; node->type = type_of(); node->value = val; + node->set_hash((uint32_t)std::hash{}(val)); return node; } diff --git a/src/Expr.h b/src/Expr.h index 5a800e7bd625..54aca89b2ebd 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -104,7 +104,7 @@ struct IRNode { */ virtual void accept(IRVisitor *v) const = 0; IRNode(IRNodeType t) - : node_type(t) { + : hash((uint32_t)t) { } virtual ~IRNode() = default; @@ -115,17 +115,34 @@ struct IRNode { */ mutable RefCount ref_count; - /** Each IR node subclass has a unique identifier. We can compare - * these values to do runtime type identification. We don't - * compile with rtti because that injects run-time type - * identification stuff everywhere (and often breaks when linking - * external libraries compiled without it), and we only want it - * for IR nodes. One might want to put this value in the vtable, - * but that adds another level of indirection, and for Exprs we - * have 32 free bits in between the ref count and the Type - * anyway, so this doesn't increase the memory footprint of an IR node. - */ - IRNodeType node_type; + /** Each IR node subclass has a unique identifier. We can compare these + * values to do runtime type identification. We don't compile with rtti + * because that injects run-time type identification stuff everywhere (and + * often breaks when linking external libraries compiled without it), and we + * only want it for IR nodes. One might want to put this value in the + * vtable, but that adds another level of indirection, and for Exprs we have + * 32 free bits in between the ref count and the Type field anyway. We use + * the first 8 to store the node type, and the next 24 as a hash of the + * children of the node, to make syntactic comparisons faster. */ + union { + IRNodeType node_type; + uint32_t hash; + }; + + /** Set hash from a combined hash of this node's arguments (see + * combine_hash below), keeping the node type intact. The low bits of a + * multiply-add hash are of poor quality, so we discard them (rather + * than shifting them up) in favor of the node type. Which end of the + * word the node type landed in when we wrote it via the node_type + * member of the union depends on the endianness of the machine. */ + HALIDE_ALWAYS_INLINE + void set_hash(uint32_t args_hash) { +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + hash = (args_hash >> 8) | ((uint32_t)node_type << 24); +#else + hash = (args_hash & 0xffffff00u) | (uint32_t)node_type; +#endif + } }; template<> @@ -163,6 +180,22 @@ struct BaseExprNode : public IRNode { Type type; }; +/** Combine one or more child hashes (or plain uint32_t fields) into a + * running hash, for use in the make() methods of Expr nodes below. Pass + * the result to IRNode::set_hash to fold in the node type and get the + * final hash - see the make() methods below for examples. */ +// @{ +HALIDE_ALWAYS_INLINE +uint32_t combine_hash(uint32_t hash, uint32_t child_hash) { + return hash * 2654435761u + child_hash; +} + +template +HALIDE_ALWAYS_INLINE uint32_t combine_hash(uint32_t hash, uint32_t child_hash, Rest... rest) { + return combine_hash(combine_hash(hash, child_hash), rest...); +} +// @} + /** We use the "curiously recurring template pattern" to avoid duplicated code in the IR Nodes. These classes live between the abstract base classes and the actual IR Nodes in the @@ -342,6 +375,12 @@ struct Expr : public Internal::IRHandle { Type type() const { return get()->type; } + + /** Get the cheap hash of this expression node. See IRNode::hash. */ + HALIDE_ALWAYS_INLINE + uint32_t hash() const { + return get()->hash; + } }; /** This lets you use an Expr as a key in a map of the form diff --git a/src/IR.cpp b/src/IR.cpp index a5ff626ed7e6..611b0af5a2c9 100644 --- a/src/IR.cpp +++ b/src/IR.cpp @@ -4,6 +4,7 @@ #include "IROperator.h" #include "IRPrinter.h" #include "IRVisitor.h" +#include #include #include @@ -44,6 +45,7 @@ Expr Cast::make(Type t, Expr v) { Cast *node = new Cast; node->type = t; + node->set_hash(combine_hash(v.hash(), t.hash())); node->value = std::move(v); return node; } @@ -60,6 +62,7 @@ Expr Reinterpret::make(Type t, Expr v) { Reinterpret *node = new Reinterpret; node->type = t; + node->set_hash(combine_hash(v.hash(), t.hash())); node->value = std::move(v); return node; } @@ -71,6 +74,7 @@ Expr Add::make(Expr a, Expr b) { Add *node = new Add; node->type = a.type(); + node->set_hash(combine_hash(a.hash(), b.hash())); node->a = std::move(a); node->b = std::move(b); return node; @@ -83,6 +87,7 @@ Expr Sub::make(Expr a, Expr b) { Sub *node = new Sub; node->type = a.type(); + node->set_hash(combine_hash(a.hash(), b.hash())); node->a = std::move(a); node->b = std::move(b); return node; @@ -95,6 +100,7 @@ Expr Mul::make(Expr a, Expr b) { Mul *node = new Mul; node->type = a.type(); + node->set_hash(combine_hash(a.hash(), b.hash())); node->a = std::move(a); node->b = std::move(b); return node; @@ -107,6 +113,7 @@ Expr Div::make(Expr a, Expr b) { Div *node = new Div; node->type = a.type(); + node->set_hash(combine_hash(a.hash(), b.hash())); node->a = std::move(a); node->b = std::move(b); return node; @@ -119,6 +126,7 @@ Expr Mod::make(Expr a, Expr b) { Mod *node = new Mod; node->type = a.type(); + node->set_hash(combine_hash(a.hash(), b.hash())); node->a = std::move(a); node->b = std::move(b); return node; @@ -131,6 +139,7 @@ Expr Min::make(Expr a, Expr b) { Min *node = new Min; node->type = a.type(); + node->set_hash(combine_hash(a.hash(), b.hash())); node->a = std::move(a); node->b = std::move(b); return node; @@ -143,6 +152,7 @@ Expr Max::make(Expr a, Expr b) { Max *node = new Max; node->type = a.type(); + node->set_hash(combine_hash(a.hash(), b.hash())); node->a = std::move(a); node->b = std::move(b); return node; @@ -155,6 +165,7 @@ Expr EQ::make(Expr a, Expr b) { EQ *node = new EQ; node->type = Bool(a.type().lanes()); + node->set_hash(combine_hash(a.hash(), b.hash())); node->a = std::move(a); node->b = std::move(b); return node; @@ -167,6 +178,7 @@ Expr NE::make(Expr a, Expr b) { NE *node = new NE; node->type = Bool(a.type().lanes()); + node->set_hash(combine_hash(a.hash(), b.hash())); node->a = std::move(a); node->b = std::move(b); return node; @@ -179,6 +191,7 @@ Expr LT::make(Expr a, Expr b) { LT *node = new LT; node->type = Bool(a.type().lanes()); + node->set_hash(combine_hash(a.hash(), b.hash())); node->a = std::move(a); node->b = std::move(b); return node; @@ -191,6 +204,7 @@ Expr LE::make(Expr a, Expr b) { LE *node = new LE; node->type = Bool(a.type().lanes()); + node->set_hash(combine_hash(a.hash(), b.hash())); node->a = std::move(a); node->b = std::move(b); return node; @@ -203,6 +217,7 @@ Expr GT::make(Expr a, Expr b) { GT *node = new GT; node->type = Bool(a.type().lanes()); + node->set_hash(combine_hash(a.hash(), b.hash())); node->a = std::move(a); node->b = std::move(b); return node; @@ -215,6 +230,7 @@ Expr GE::make(Expr a, Expr b) { GE *node = new GE; node->type = Bool(a.type().lanes()); + node->set_hash(combine_hash(a.hash(), b.hash())); node->a = std::move(a); node->b = std::move(b); return node; @@ -229,6 +245,7 @@ Expr And::make(Expr a, Expr b) { And *node = new And; node->type = Bool(a.type().lanes()); + node->set_hash(combine_hash(a.hash(), b.hash())); node->a = std::move(a); node->b = std::move(b); return node; @@ -243,6 +260,7 @@ Expr Or::make(Expr a, Expr b) { Or *node = new Or; node->type = Bool(a.type().lanes()); + node->set_hash(combine_hash(a.hash(), b.hash())); node->a = std::move(a); node->b = std::move(b); return node; @@ -254,6 +272,7 @@ Expr Not::make(Expr a) { Not *node = new Not; node->type = Bool(a.type().lanes()); + node->set_hash(combine_hash(a.hash(), 0)); node->a = std::move(a); return node; } @@ -269,6 +288,8 @@ Expr Select::make(Expr condition, Expr true_value, Expr false_value) { Select *node = new Select; node->type = true_value.type(); + node->set_hash(combine_hash(condition.hash(), + true_value.hash(), false_value.hash())); node->condition = std::move(condition); node->true_value = std::move(true_value); node->false_value = std::move(false_value); @@ -285,6 +306,8 @@ Expr Load::make(Type type, const std::string &name, Expr index, Buffer<> image, Load *node = new Load; node->type = type; node->name = name; + node->set_hash(combine_hash((uint32_t)std::hash{}(name), + index.hash(), predicate.hash())); node->predicate = std::move(predicate); node->index = std::move(index); node->image = std::move(image); @@ -320,6 +343,8 @@ Expr Ramp::make(Expr base, Expr stride, int lanes) { Ramp *node = new Ramp; node->type = base.type().with_lanes(lanes * base.type().lanes()); + node->set_hash(combine_hash((uint32_t)lanes, + base.hash(), stride.hash())); node->base = std::move(base); node->stride = std::move(stride); node->lanes = lanes; @@ -332,6 +357,7 @@ Expr Broadcast::make(Expr value, int lanes) { Broadcast *node = new Broadcast; node->type = value.type().with_lanes(lanes * value.type().lanes()); + node->set_hash(combine_hash((uint32_t)lanes, value.hash())); node->value = std::move(value); node->lanes = lanes; return node; @@ -344,6 +370,8 @@ Expr Let::make(const std::string &name, Expr value, Expr body) { Let *node = new Let; node->type = body.type(); node->name = name; + node->set_hash(combine_hash((uint32_t)std::hash{}(name), + value.hash(), body.hash())); node->value = std::move(value); node->body = std::move(body); return node; @@ -974,6 +1002,12 @@ Expr Call::make(Type type, const std::string &name, const std::vector &arg Call *node = new Call; node->type = type; node->name = name; + uint32_t h = combine_hash((uint32_t)std::hash{}(name), + (uint32_t)call_type, (uint32_t)value_index); + for (const auto &arg : args) { + h = combine_hash(h, arg.hash()); + } + node->set_hash(h); node->args = args; node->call_type = call_type; node->func = std::move(func); @@ -995,6 +1029,7 @@ Expr Variable::make(Type type, const std::string &name, Buffer<> image, Paramete Variable *node = new Variable; node->type = type; node->name = name; + node->set_hash((uint32_t)std::hash{}(name)); node->image = std::move(image); node->param = std::move(param); node->reduction_domain = std::move(reduction_domain); @@ -1017,6 +1052,14 @@ Expr Shuffle::make(const std::vector &vectors, Shuffle *node = new Shuffle; node->type = element_ty.with_lanes((int)indices.size()); + uint32_t h = 0; + for (int i : indices) { + h = combine_hash(h, (uint32_t)i); + } + for (const auto &v : vectors) { + h = combine_hash(h, v.hash()); + } + node->set_hash(h); node->vectors = vectors; node->indices = indices; return node; @@ -1259,6 +1302,7 @@ Expr VectorReduce::make(VectorReduce::Operator op, << lanes << " " << vec.type().lanes() << "\n"; VectorReduce *node = new VectorReduce; node->type = vec.type().with_lanes(lanes); + node->set_hash(combine_hash((uint32_t)op, vec.hash())); node->op = op; node->value = std::move(vec); return node; diff --git a/src/IREquality.h b/src/IREquality.h index c6987f873c4e..7b33d000b485 100644 --- a/src/IREquality.h +++ b/src/IREquality.h @@ -38,7 +38,11 @@ HALIDE_ALWAYS_INLINE bool equal(const IRNode &a, const IRNode &b) { if (&a == &b) { return true; - } else if (a.node_type != b.node_type) { + } else if (a.hash != b.hash) { + // IRNode::hash packs the node type into its low 8 bits, so a + // mismatch here also covers the a.node_type != b.node_type case. + // Equal nodes always have equal hashes, so this lets us skip the + // full recursive comparison below. return false; } else { return equal_impl(a, b); @@ -63,7 +67,7 @@ HALIDE_ALWAYS_INLINE bool graph_equal(const IRNode &a, const IRNode &b) { if (&a == &b) { return true; - } else if (a.node_type != b.node_type) { + } else if (a.hash != b.hash) { return false; } else { return graph_equal_impl(a, b); @@ -89,8 +93,11 @@ HALIDE_ALWAYS_INLINE bool less_than(const IRNode &a, const IRNode &b) { if (&a == &b) { return false; - } else if (a.node_type < b.node_type) { - return true; + } else if (a.hash != b.hash) { + // This ordering is arbitrary (it's just used for map keys), so we're + // free to use the cheap hash to distinguish unequal nodes instead of + // doing a full comparison. + return a.hash < b.hash; } else { return less_than_impl(a, b); } @@ -118,8 +125,8 @@ HALIDE_ALWAYS_INLINE bool graph_less_than(const IRNode &a, const IRNode &b) { if (&a == &b) { return false; - } else if (a.node_type < b.node_type) { - return true; + } else if (a.hash != b.hash) { + return a.hash < b.hash; } else { return graph_less_than_impl(a, b); } diff --git a/src/IRMatch.h b/src/IRMatch.h index 6fa4cadc4eae..5711b3b534be 100644 --- a/src/IRMatch.h +++ b/src/IRMatch.h @@ -490,6 +490,13 @@ struct Wild { return state.get_binding(i); } + // The bound node itself. Unlike make() this doesn't even touch a reference + // count, which lets predicates inspect what matched for free. + HALIDE_ALWAYS_INLINE + const BaseExprNode *bound_node(MatcherState &state) const noexcept { + return state.get_binding(i); + } + constexpr static bool foldable = false; }; @@ -2554,7 +2561,7 @@ struct CanProve { // Includes a raw call to an inlined make method, so don't inline. [[nodiscard]] HALIDE_NEVER_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const { Expr condition = a.make(state, {}); - condition = prover->mutate(condition, nullptr); + condition = prover->simplify_can_prove_condition(condition); val.u.u64 = is_const_one(condition); ty = Bool(condition.type().lanes()); return false; @@ -2573,6 +2580,119 @@ std::ostream &operator<<(std::ostream &s, const CanProve &op) { return s; } +// Like can_prove, but only looks the condition up in the facts the prover +// already knows, instead of recursively invoking it. Much cheaper, and it +// cannot recurse, so unlike can_prove it is safe in a rule whose left-hand +// side matches expressions the prover may construct while proving it. +template +struct KnownTrue { + struct pattern_tag {}; + A a; + Prover *prover; // An existing simplifying mutator + + constexpr static uint32_t binds = bindings::mask; + + // This rule is a boolean-valued predicate. Bools have type UIntImm. + constexpr static IRNodeType min_node_type = IRNodeType::UIntImm; + constexpr static IRNodeType max_node_type = IRNodeType::UIntImm; + constexpr static bool canonical = true; + + constexpr static bool foldable = true; + + // Includes a raw call to an inlined make method, so don't inline. + [[nodiscard]] HALIDE_NEVER_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const { + Expr condition = a.make(state, {}); + val.u.u64 = prover->is_known_true(condition) ? 1 : 0; + ty = Bool(condition.type().lanes()); + return false; + } +}; + +template +HALIDE_ALWAYS_INLINE auto known_true(A &&a, Prover *p) noexcept -> KnownTrue { + assert_is_lvalue_if_expr(); + return {pattern_arg(a), p}; +} + +template +std::ostream &operator<<(std::ostream &s, const KnownTrue &op) { + s << "known_true(" << op.a << ")"; + return s; +} + +// Detects patterns that can hand back the node they matched without building +// anything. The predicates below are restricted to these, which is what makes +// them allocation-free: it is a compile error to ask about a derived expression +// like min_diff(x, y + 1). Put the offset on the other side of the comparison +// instead: min_diff(x, y) >= 1. +template +struct has_bound_node : std::false_type {}; + +template +struct has_bound_node().bound_node(std::declval()))>> + : std::true_type {}; + +// Bounds on the difference between two matched expressions, derived from the +// facts the prover has learned. Used as (min_diff(x, y, this) >= 0) and +// friends. When nothing is known the fold reports overflow, which the rewriter +// already treats as a failed predicate, so the rule simply doesn't fire. +template +struct DiffBound { + struct pattern_tag {}; + A a; + B b; + Prover *prover; + + static_assert(has_bound_node::value && has_bound_node::value, + "The operands of min_diff/max_diff must be wildcards, so that " + "testing the predicate doesn't have to construct any IR."); + + constexpr static uint32_t binds = bindings::mask | bindings::mask; + + // This is an integer-valued term of a comparison. + constexpr static IRNodeType min_node_type = IRNodeType::IntImm; + constexpr static IRNodeType max_node_type = IRNodeType::IntImm; + constexpr static bool canonical = true; + + constexpr static bool foldable = true; + + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const noexcept { + int64_t result = 0; + bool known; + if (is_min) { + known = prover->known_min_diff(a.bound_node(state), b.bound_node(state), &result); + } else { + known = prover->known_max_diff(a.bound_node(state), b.bound_node(state), &result); + } + val.u.i64 = result; + ty = Int(64); + // Report an unknown bound as an overflow, which fails the predicate. + return !known; + } +}; + +template +HALIDE_ALWAYS_INLINE auto min_diff(A &&a, B &&b, Prover *p) noexcept + -> DiffBound { + assert_is_lvalue_if_expr(); + assert_is_lvalue_if_expr(); + return {pattern_arg(a), pattern_arg(b), p}; +} + +template +HALIDE_ALWAYS_INLINE auto max_diff(A &&a, B &&b, Prover *p) noexcept + -> DiffBound { + assert_is_lvalue_if_expr(); + assert_is_lvalue_if_expr(); + return {pattern_arg(a), pattern_arg(b), p}; +} + +template +std::ostream &operator<<(std::ostream &s, const DiffBound &op) { + s << (is_min ? "min_diff(" : "max_diff(") << op.a << ", " << op.b << ")"; + return s; +} + template struct IsFloat { struct pattern_tag {}; diff --git a/src/Lower.cpp b/src/Lower.cpp index 21acfff8df2d..16247d211b55 100644 --- a/src/Lower.cpp +++ b/src/Lower.cpp @@ -305,6 +305,11 @@ void lower_impl(const vector &output_funcs, s = storage_flattening(s, outputs, env, t); log("Lowering after storage flattening:", s); + // Every pass that reads a region or an allocation size out of the IR has + // now run, so from here a clamp is only worth what its value is worth, and + // the simplifier may use what it knows to remove a redundant one. + ScopedRegionsInferred regions_inferred; + debug(1) << "Adding atomic mutex allocation...\n"; s = add_atomic_mutex(s, outputs); log("Lowering after adding atomic mutex allocation:", s); diff --git a/src/Simplify.cpp b/src/Simplify.cpp index 18129614aca7..d29b4ddd37c8 100644 --- a/src/Simplify.cpp +++ b/src/Simplify.cpp @@ -84,7 +84,141 @@ void Simplify::found_buffer_reference(const string &name, size_t dimensions) { } } +namespace { + +// Rewrite (a - b) as (a' - b') + offset by stripping constant terms off either +// side, so that a fact about x and y + 3 and a query about x and y meet at the +// same pair. Walks the existing nodes; builds nothing. +void peel_constant_offsets(const BaseExprNode *&a, const BaseExprNode *&b, int64_t &offset) { + // Peels one constant term off e if there is one, returning whether it did. + // The constant is added to delta, which the caller applies with the sign + // appropriate to the side e is on. + auto peel_one = [](const BaseExprNode *&e, int64_t &delta) { + if (e->node_type == IRNodeType::Add) { + const Add *add = (const Add *)e; + if (const IntImm *i = add->b.as()) { + if (add_would_overflow(64, delta, i->value)) { + return false; + } + delta += i->value; + e = add->a.get(); + return true; + } else if (const IntImm *i = add->a.as()) { + if (add_would_overflow(64, delta, i->value)) { + return false; + } + delta += i->value; + e = add->b.get(); + return true; + } + } else if (e->node_type == IRNodeType::Sub) { + const Sub *sub = (const Sub *)e; + if (const IntImm *i = sub->b.as()) { + if (sub_would_overflow(64, delta, i->value)) { + return false; + } + delta -= i->value; + e = sub->a.get(); + return true; + } + } + return false; + }; + + // A constant on the left of the difference adds to the offset; one on the + // right subtracts from it, so accumulate it negated and subtract at the end. + int64_t from_a = 0, from_b = 0; + while (peel_one(a, from_a)) { + } + while (peel_one(b, from_b)) { + } + if (!sub_would_overflow(64, from_a, from_b)) { + offset = from_a - from_b; + } else { + offset = 0; + } +} + +} // namespace + +namespace { +// Lowering is single-threaded per pipeline, but several pipelines can be +// lowered at once, so this is per-thread rather than global. +thread_local bool t_regions_have_been_inferred = false; +} // namespace + +bool regions_have_been_inferred() { + return t_regions_have_been_inferred; +} + +ScopedRegionsInferred::ScopedRegionsInferred() + : old_value(t_regions_have_been_inferred) { + t_regions_have_been_inferred = true; +} + +ScopedRegionsInferred::~ScopedRegionsInferred() { + t_regions_have_been_inferred = old_value; +} + +void Simplify::ScopedFact::learn_difference(const Expr &a, const Expr &b, + const ConstantInterval &diff, bool invert) { + // Nothing may be ordered from a fact until lowering has finished reading + // regions and allocation sizes out of the IR. A clamp around an index is + // part of how those are derived, so removing one on the strength of + // something we happen to know leaves the region asked for as wide as the + // unclamped index could reach. + if (!regions_have_been_inferred()) { + return; + } + // Differences are only meaningful where they can't wrap. + if (!simplify->no_overflow_int(a.type()) || a.type() != b.type()) { + return; + } + + const BaseExprNode *pa = a.get(), *pb = b.get(); + int64_t offset = 0; + peel_constant_offsets(pa, pb, offset); + + // (a - b) = (pa - pb) + offset, so the bound on the peeled pair is the + // bound we were given shifted the other way. + ConstantInterval peeled = diff - offset; + if (invert && !peeled.is_single_point()) { + // Only a single removed point is representable. + return; + } + + simplify->add_difference_key(Simplify::difference_key(pa->hash, pb->hash)); + simplify->known_bounds.push_back( + Simplify::KnownBound{Expr(pa), Expr(pb), peeled, invert}); +} + void Simplify::ScopedFact::learn_false(const Expr &fact) { + // Canonicalize the direction of comparisons, so that facts are stored in + // the same form the simplifier produces when it visits them. + if (const GT *gt = fact.as()) { + learn_false(gt->b < gt->a); + return; + } else if (const GE *ge = fact.as()) { + learn_false(!(ge->a < ge->b)); + return; + } + + // Record what this says about the difference between the two sides. And, + // Not, and the tag intrinsic are handled by the recursion below instead. + if (const LT *lt = fact.as()) { + // !(a < b) -> a - b >= 0 + learn_difference(lt->a, lt->b, ConstantInterval::bounded_below(0), false); + } else if (const LE *le = fact.as()) { + // !(a <= b) -> a - b >= 1 + learn_difference(le->a, le->b, ConstantInterval::bounded_below(1), false); + } else if (const EQ *eq = fact.as()) { + // !(a == b) -> a - b is anything but zero + learn_difference(eq->a, eq->b, ConstantInterval::single_point(0), true); + } else if (const NE *ne = fact.as()) { + // !(a != b) -> a - b == 0 + learn_difference(ne->a, ne->b, ConstantInterval::single_point(0), false); + } + Simplify::VarInfo info; info.old_uses = info.new_uses = 0; if (const Variable *v = fact.as()) { @@ -172,6 +306,32 @@ void Simplify::ScopedFact::learn_lower_bound(const Variable *v, int64_t val) { } void Simplify::ScopedFact::learn_true(const Expr &fact) { + // Canonicalize the direction of comparisons, so that facts are stored in + // the same form the simplifier produces when it visits them. + if (const GT *gt = fact.as()) { + learn_true(gt->b < gt->a); + return; + } else if (const GE *ge = fact.as()) { + learn_true(!(ge->a < ge->b)); + return; + } + + // Record what this says about the difference between the two sides. And, + // Not, and the tag intrinsic are handled by the recursion below instead. + if (const LT *lt = fact.as()) { + // a < b -> a - b <= -1 + learn_difference(lt->a, lt->b, ConstantInterval::bounded_above(-1), false); + } else if (const LE *le = fact.as()) { + // a <= b -> a - b <= 0 + learn_difference(le->a, le->b, ConstantInterval::bounded_above(0), false); + } else if (const EQ *eq = fact.as()) { + // a == b -> a - b == 0 + learn_difference(eq->a, eq->b, ConstantInterval::single_point(0), false); + } else if (const NE *ne = fact.as()) { + // a != b -> a - b is anything but zero + learn_difference(ne->a, ne->b, ConstantInterval::single_point(0), true); + } + Simplify::VarInfo info; info.old_uses = info.new_uses = 0; if (const Variable *v = fact.as()) { @@ -345,16 +505,56 @@ void Simplify::ScopedFact::learn_true(const Expr &fact) { } namespace { +// Is a boolean Expr known to be true or false? Facts are stored in the same +// form the simplifier itself produces, so a comparison has to be canonicalized +// the same way before looking it up. +std::optional lookup_fact(const Expr &e, + const std::set &truths, + const std::set &falsehoods) { + if (const Not *n = e.as()) { + auto known = lookup_fact(n->a, truths, falsehoods); + return known ? std::make_optional(!*known) : known; + } else if (const GT *gt = e.as()) { + return lookup_fact(gt->b < gt->a, truths, falsehoods); + } else if (const GE *ge = e.as()) { + return lookup_fact(!(ge->a < ge->b), truths, falsehoods); + } + + if (truths.count(e)) { + return true; + } else if (falsehoods.count(e)) { + return false; + } + + // A comparison may also be settled by the other strictness of the same + // comparison, in either direction. + if (const LT *lt = e.as()) { + // a < b is implied by !(b <= a), and ruled out by b <= a and by b < a. + if (falsehoods.count(lt->b <= lt->a)) { + return true; + } else if (truths.count(lt->b <= lt->a) || truths.count(lt->b < lt->a)) { + return false; + } + } else if (const LE *le = e.as()) { + // a <= b is implied by a < b and by !(b < a), and ruled out by b < a. + if (truths.count(le->a < le->b) || falsehoods.count(le->b < le->a)) { + return true; + } else if (truths.count(le->b < le->a)) { + return false; + } + } + + return std::nullopt; +} + template T substitute_facts_impl(const T &t, const std::set &truths, const std::set &falsehoods) { return mutate_with(t, [&](auto *self, const Expr &e) { if (e.type().is_bool()) { - if (truths.count(e)) { - return make_one(e.type()); - } else if (falsehoods.count(e)) { - return make_zero(e.type()); + if (auto known = lookup_fact(e, truths, falsehoods)) { + return *known ? make_one(e.type()) : make_zero(e.type()); } } return self->mutate_base(e); @@ -370,13 +570,228 @@ Stmt Simplify::ScopedFact::substitute_facts(const Stmt &s) { return substitute_facts_impl(s, truths, falsehoods); } +namespace { + +// Intersect acc with d, reporting whether the result would be empty rather than +// constructing it. make_intersection asserts on an empty result, and empty means +// the facts contradict each other, which means this code is unreachable. We +// don't try to exploit that here; we just decline to tighten any further. +bool intersect_if_nonempty(ConstantInterval &acc, const ConstantInterval &d) { + ConstantInterval result = acc; + if (d.min_defined && (!result.min_defined || d.min > result.min)) { + result.min = d.min; + result.min_defined = true; + } + if (d.max_defined && (!result.max_defined || d.max < result.max)) { + result.max = d.max; + result.max_defined = true; + } + if (result.min_defined && result.max_defined && result.min > result.max) { + return false; + } + acc = result; + return true; +} + +// What the shape of the two sides says about (a - b) on its own, with no facts +// involved: a min is at most either of its operands, and a max is at least +// either of them. Only the immediate operands are inspected, so this stays a +// couple of pointer comparisons rather than a search. +ConstantInterval structural_difference(const BaseExprNode *a, const BaseExprNode *b) { + ConstantInterval result; + + // Same restriction as learning a fact: a difference only means what we take + // it to mean for integers that don't wrap. It keeps floats, where a NaN + // makes even min(p, q) <= p false, out of it too. + if (!(a->type.is_int() && a->type.bits() >= 32) || a->type != b->type) { + return result; + } + + auto is_operand_of = [](const BaseExprNode *e, const BaseExprNode *node) { + if (node->node_type == IRNodeType::Min) { + const Min *m = (const Min *)node; + return equal(*m->a.get(), *e) || equal(*m->b.get(), *e); + } else if (node->node_type == IRNodeType::Max) { + const Max *m = (const Max *)node; + return equal(*m->a.get(), *e) || equal(*m->b.get(), *e); + } + return false; + }; + + // min(p, q) - b <= 0 and max(p, q) - b >= 0, when b is one of the operands. + if (a->node_type == IRNodeType::Min && is_operand_of(b, a)) { + result = ConstantInterval::bounded_above(0); + } else if (a->node_type == IRNodeType::Max && is_operand_of(b, a)) { + result = ConstantInterval::bounded_below(0); + } else if (b->node_type == IRNodeType::Min && is_operand_of(a, b)) { + // a - min(p, q) >= 0 + result = ConstantInterval::bounded_below(0); + } else if (b->node_type == IRNodeType::Max && is_operand_of(a, b)) { + result = ConstantInterval::bounded_above(0); + } + + return result; +} + +} // namespace + +ConstantInterval Simplify::known_difference(const BaseExprNode *a, const BaseExprNode *b) { + ConstantInterval result; + + // Canonicalize the query the way the facts were canonicalized when learned. + int64_t offset = 0; + peel_constant_offsets(a, b, offset); + + if (equal(*a, *b)) { + result = ConstantInterval::single_point(0); + } else { + if (a->node_type == IRNodeType::IntImm && b->node_type == IRNodeType::IntImm && + !sub_would_overflow(64, ((const IntImm *)a)->value, ((const IntImm *)b)->value)) { + // Two constants need no facts to compare. + result = ConstantInterval::single_point(((const IntImm *)a)->value - + ((const IntImm *)b)->value); + } else { + intersect_if_nonempty(result, structural_difference(a, b)); + } + } + + if (!result.is_single_point() && !known_bounds.empty()) { + // A hole only tightens the bounds once we know where the ends are, so + // collect them as we go and apply them below. There are hardly ever any. + constexpr int max_holes = 4; + int64_t holes[max_holes]; + int num_holes = 0; + + const uint32_t fa = a->hash, fb = b->hash; + // One test against the whole table before looking at any record. + if (!difference_key_present(difference_key(fa, fb))) { + result += offset; + return result; + } + for (const KnownBound &kb : known_bounds) { + // Reject on the hashes first: a record about some other pair costs + // a pair of integer compares rather than a walk over two Exprs. + const uint32_t kba = kb.a.get()->hash, kbb = kb.b.get()->hash; + const bool same_order = (fa == kba && fb == kbb); + const bool swapped = (fa == kbb && fb == kba); + if (!same_order && !swapped) { + continue; + } + + ConstantInterval d; + if (same_order && equal(*a, *kb.a.get()) && equal(*b, *kb.b.get())) { + d = kb.diff; + } else if (swapped && equal(*a, *kb.b.get()) && equal(*b, *kb.a.get())) { + // We know about (b - a), and this is the other direction. + d = -kb.diff; + } else { + continue; + } + + if (kb.invert) { + if (num_holes < max_holes) { + holes[num_holes++] = d.min; + } + } else if (!intersect_if_nonempty(result, d)) { + break; + } + } + + for (int i = 0; i < num_holes; i++) { + const int64_t hole = holes[i]; + // Removing a point only narrows the bounds if it is at one end, + // and only if something is left afterwards: a hole that swallows + // the whole interval means the facts contradict each other, so the + // code is unreachable. Say nothing rather than describe an empty + // set with a backwards interval. + if (result.min_defined && result.max_defined && + result.min == hole && result.max == hole) { + continue; + } + if (result.min_defined && result.min == hole && + !add_would_overflow(64, hole, 1)) { + result.min = hole + 1; + } + if (result.max_defined && result.max == hole && + !sub_would_overflow(64, hole, 1)) { + result.max = hole - 1; + } + } + } + + // Undo the canonicalization: (a - b) = (peeled a - peeled b) + offset. + result += offset; + + return result; +} + +bool Simplify::known_min_diff(const BaseExprNode *a, const BaseExprNode *b, int64_t *result) { + ConstantInterval bounds = known_difference(a, b); + if (bounds.min_defined) { + *result = bounds.min; + return true; + } + return false; +} + +bool Simplify::known_max_diff(const BaseExprNode *a, const BaseExprNode *b, int64_t *result) { + ConstantInterval bounds = known_difference(a, b); + if (bounds.max_defined) { + *result = bounds.max; + return true; + } + return false; +} + +bool Simplify::is_known_true(const Expr &e) { + if (truths.empty() && falsehoods.empty()) { + return false; + } + auto known = lookup_fact(e, truths, falsehoods); + return known && *known; +} + +Expr Simplify::simplify_can_prove_condition(const Expr &e) { + if (can_prove_depth >= max_can_prove_depth) { + // Too deep to safely recurse into the full simplifier. The only thing + // the caller does with the result is check whether it is the literal + // constant true, and nothing here can fold a compound expression (an + // And of two known-true operands stays an unfolded And, not true) -- + // that folding is exactly the recursive work we're declining to do. + // So a substitute_facts tree walk can't prove anything a direct + // lookup of the condition itself couldn't already: skip the walk. + if (is_known_true(e)) { + return const_true(e.type().lanes(), nullptr); + } + return e; + } + ScopedValue guard(can_prove_depth, can_prove_depth + 1); + return mutate(substitute_facts(e), nullptr); +} + +Expr Simplify::substitute_facts(const Expr &e) { + if (truths.empty() && falsehoods.empty()) { + return e; + } + return substitute_facts_impl(e, truths, falsehoods); +} + Simplify::ScopedFact::~ScopedFact() { + if (!simplify) { + // Moved from; the object that took over owns the cleanup. + return; + } for (const auto *v : pop_list) { simplify->var_info.pop(v->name); } for (const auto *v : bounds_pop_list) { simplify->bounds_and_alignment_info.pop(v->name); } + internal_assert(simplify->known_bounds.size() >= known_bounds_size); + simplify->known_bounds.resize(known_bounds_size); + for (int i = 0; i < Simplify::difference_key_words; i++) { + simplify->difference_keys[i] = saved_difference_keys[i]; + } for (const auto &e : truths) { simplify->truths.erase(e); } diff --git a/src/Simplify.h b/src/Simplify.h index 64459a43c44b..e4bbc7c40946 100644 --- a/src/Simplify.h +++ b/src/Simplify.h @@ -34,6 +34,25 @@ Expr simplify(const Expr &, /** Attempt to statically prove an expression is true using the simplifier. */ bool can_prove(Expr e, const Scope &bounds = Scope::empty_scope()); +/** Has lowering finished deriving regions and allocation sizes from the IR? + * + * A clamp around an index is not only a statement about a value: it is part of + * how those are derived. Until they have been, the simplifier must not use a + * condition it happens to know to remove one, or the region asked for grows to + * whatever the unclamped index could reach. Afterwards the derived regions are + * already IR of their own, and removing a redundant clamp is just a + * simplification. */ +bool regions_have_been_inferred(); + +/** Mark regions as derived for the rest of the enclosing scope. Lowering does + * this once, after the last pass that reads a region out of the IR. */ +struct ScopedRegionsInferred { + bool old_value; + ScopedRegionsInferred(); + ~ScopedRegionsInferred(); + ScopedRegionsInferred(const ScopedRegionsInferred &) = delete; +}; + /** Simplify expressions found in a statement, but don't simplify * across different statements. This is safe to perform at an earlier * stage in lowering than full simplification of a stmt. */ diff --git a/src/Simplify_Div.cpp b/src/Simplify_Div.cpp index 4098f6f027e7..d7839340fb99 100644 --- a/src/Simplify_Div.cpp +++ b/src/Simplify_Div.cpp @@ -84,6 +84,18 @@ Expr Simplify::visit(const Div *op, ExprInfo *info) { rewrite(select(x, c0, c1) / c2, select(x, fold(c0 / c2), fold(c1 / c2))) || (!op->type.is_float() && rewrite(x / x, select(x == 0, 0, 1))) || + + (no_overflow(op->type) && + // Facts learned higher up in the IR may tell us which side of a max + // or min survives the division. Test them early on to prevents rewrites below + // that would make it impossible to recognize the form. + (has_facts() && + (rewrite(max(x * c0, y) / c0, x, c0 > 0 && known_true(x >= y / c0, this)) || + rewrite(max(y, x * c0) / c0, x, c0 > 0 && known_true(x >= y / c0, this)) || + rewrite(min(x * c0, y) / c0, x, c0 > 0 && known_true(x <= y / c0, this)) || + rewrite(min(y, x * c0) / c0, x, c0 > 0 && known_true(x <= y / c0, this)) || + false))) || + (no_overflow(op->type) && // Fold repeated division (rewrite((x / c0) / c2, x / fold(c0 * c2), c0 > 0 && c2 > 0 && !overflows(c0 * c2)) || diff --git a/src/Simplify_Internal.h b/src/Simplify_Internal.h index 94a50bebc644..e63ea11307d2 100644 --- a/src/Simplify_Internal.h +++ b/src/Simplify_Internal.h @@ -441,30 +441,163 @@ class Simplify : public VariadicVisitor { std::set truths, falsehoods; + /** What we know about the difference between a pair of Exprs. Every + * comparison we can learn from is a statement about (a - b): a < b means it + * is at most -1, !(a < b) means it is at least 0, a == b means it is zero. + * Because the complement of a half-line is a half-line, only the negation + * of an equality fails to be an interval, and that is always a single point + * removed, which is what invert represents. */ + struct KnownBound { + Expr a, b; + ConstantInterval diff; + // If set, a - b is known *not* to lie in diff, which is always a single + // point. Only a != b (or !(a == b)) produces one of these. + bool invert = false; + }; + std::vector known_bounds; + + // A bit per pair key, over every record in the table. A query whose bit is + // clear cannot match anything, which is the answer almost every query gets. + // Wide enough that a few dozen facts leave it sparse: at 64 bits a typical + // table saturates and lets four queries in ten through to the scan. + static constexpr int difference_key_words = 4; + uint64_t difference_keys[difference_key_words] = {0}; + + /** Everything the facts tell us about (a - b), without building any IR. + * The arguments are borrowed, so this is safe to call with the raw nodes a + * rewrite rule has bound to its wildcards. */ + ConstantInterval known_difference(const BaseExprNode *a, const BaseExprNode *b); + + // Helpers over known_difference, for use as rewrite rule predicates. They + // return false when nothing is known, so that a rule asking for a bound it + // can't get simply doesn't fire. + bool known_min_diff(const BaseExprNode *a, const BaseExprNode *b, int64_t *result); + bool known_max_diff(const BaseExprNode *a, const BaseExprNode *b, int64_t *result); + + // How deeply are we nested inside the conditions of can_prove predicates? + // Proving such a condition recursively invokes the simplifier on it, so a + // rule whose left-hand side also matches something built while proving its + // own predicate recurses without bound. Bound it. + // + // The work grows sharply with this limit -- on an adversarial nest of + // min(x, y) - min(z, w) it is roughly 0.02s at 1 or 2, 0.11s at 3 and 0.72s + // at 4 -- while no rule needs the depth: instrumenting every correctness + // test shows the deepest nesting any of them reaches is one. So this is + // already a level of headroom over anything observed. + int can_prove_depth = 0; + static constexpr int max_can_prove_depth = 2; + + // Is there anything a known_true predicate could look up? Used to gate rules + // whose predicates are only ever provable from facts learned higher up in + // the IR, so that we don't pay for them in the common case. + bool has_facts() const { + return !truths.empty() || !falsehoods.empty(); + } + + // Is there anything a min_diff or max_diff predicate could look up? Only a + // comparison of non-overflowing integers leaves a record here, so this is + // strictly narrower than has_facts: a boolean fact, or a fact about a type + // that can wrap, satisfies that one while leaving this table empty. Rules + // that ask about differences must gate on this, or they spend a lookup on + // a table that cannot answer. + bool has_difference_facts() const { + return !known_bounds.empty(); + } + + // Symmetric key for a pair. Xoring two equal hashes gives zero whatever + // they were, so key that case by the hash itself rather than letting every + // pair of equal-hashing operands share the one bit. + HALIDE_ALWAYS_INLINE + static uint32_t difference_key(uint32_t fa, uint32_t fb) { + return fa == fb ? fa * 0x9e3779b9u : (fa ^ fb); + } + + // One bit per pair key. Which bit has to come from mixed bits rather than + // from the bottom of the key: an Expr's hash carries its node type in the + // low bits, so indexing by those puts every pair of the same two kinds on + // one bit, and a few dozen facts then light only a handful of them. + HALIDE_ALWAYS_INLINE + static uint32_t difference_key_bit_index(uint32_t key) { + constexpr int bits = 8; // log2(difference_key_words * 64) + static_assert(difference_key_words * 64 == (1 << bits)); + return (key * 0x9e3779b9u) >> (32 - bits); + } + + HALIDE_ALWAYS_INLINE + bool difference_key_present(uint32_t key) const { + const uint32_t bit = difference_key_bit_index(key); + return (difference_keys[bit / 64] >> (bit % 64)) & 1; + } + + HALIDE_ALWAYS_INLINE + void add_difference_key(uint32_t key) { + const uint32_t bit = difference_key_bit_index(key); + difference_keys[bit / 64] |= (uint64_t)1 << (bit % 64); + } + + // Replace exprs known to be truths or falsehoods with const_true or + // const_false. Used to inject everything currently known into the + // conditions of can_prove predicates in rewrite rules. + Expr substitute_facts(const Expr &e); + + // Simplify the condition of a can_prove predicate in a rewrite rule, using + // everything currently known. + Expr simplify_can_prove_condition(const Expr &e); + + // Is a boolean Expr already known to be true? Unlike can_prove this only + // looks the condition up in the facts, without simplifying anything. + bool is_known_true(const Expr &e); + struct ScopedFact { Simplify *simplify; std::vector pop_list; std::vector bounds_pop_list; std::set truths, falsehoods; + // Everything in the simplifier's known_bounds from this index on was + // pushed by this scope, and is truncated away again when it ends. + size_t known_bounds_size = 0; + // Bits can't be cleared one at a time, so keep the summary from before + // this scope and put it back wholesale. + uint64_t saved_difference_keys[difference_key_words] = {0}; void learn_false(const Expr &fact); void learn_true(const Expr &fact); void learn_upper_bound(const Variable *v, int64_t val); void learn_lower_bound(const Variable *v, int64_t val); + // Record what a comparison says about the difference between its sides. + void learn_difference(const Expr &a, const Expr &b, const ConstantInterval &diff, bool invert); // Replace exprs known to be truths or falsehoods with const_true or const_false. Expr substitute_facts(const Expr &e); Stmt substitute_facts(const Stmt &s); ScopedFact(Simplify *s) - : simplify(s) { + : simplify(s), known_bounds_size(s->known_bounds.size()) { + for (int i = 0; i < difference_key_words; i++) { + saved_difference_keys[i] = s->difference_keys[i]; + } } ~ScopedFact(); // allow move but not copy ScopedFact(const ScopedFact &that) = delete; - ScopedFact(ScopedFact &&that) = default; + // Not defaulted: the moved-from object must not undo anything in its + // destructor. The containers below would be empty after a move and so + // would be harmless, but known_bounds_size would survive and truncate + // away the facts this scope had just learned. + ScopedFact(ScopedFact &&that) noexcept + : simplify(that.simplify), + pop_list(std::move(that.pop_list)), + bounds_pop_list(std::move(that.bounds_pop_list)), + truths(std::move(that.truths)), + falsehoods(std::move(that.falsehoods)), + known_bounds_size(that.known_bounds_size) { + for (int i = 0; i < difference_key_words; i++) { + saved_difference_keys[i] = that.saved_difference_keys[i]; + } + that.simplify = nullptr; + } }; // Tell the simplifier to learn from and exploit a boolean diff --git a/src/Simplify_Max.cpp b/src/Simplify_Max.cpp index 88d3ce2cbf5e..718ddfd30ac7 100644 --- a/src/Simplify_Max.cpp +++ b/src/Simplify_Max.cpp @@ -71,6 +71,10 @@ Expr Simplify::visit(const Max *op, ExprInfo *info) { // RHS for ExprInfo to update correctly. if (EVAL_IN_LAMBDA // (rewrite(max(x, x), a) || + // Facts learned higher up in the IR may tell us which side wins. + (has_difference_facts() && + (rewrite(max(x, y), a, min_diff(x, y, this) >= 0) || + rewrite(max(x, y), b, max_diff(x, y, this) <= 0))) || rewrite(max(x, c0), b, is_max_value(c0)) || rewrite(max(x, c0), a, is_min_value(c0)) || rewrite(max((x / c0) * c0, x), b, c0 > 0) || diff --git a/src/Simplify_Min.cpp b/src/Simplify_Min.cpp index 5203a0c14166..0ee489c4bb28 100644 --- a/src/Simplify_Min.cpp +++ b/src/Simplify_Min.cpp @@ -70,6 +70,10 @@ Expr Simplify::visit(const Min *op, ExprInfo *info) { // RHS for ExprInfo to update correctly. if (EVAL_IN_LAMBDA // (rewrite(min(x, x), a) || + // Facts learned higher up in the IR may tell us which side wins. + (has_difference_facts() && + (rewrite(min(x, y), a, max_diff(x, y, this) <= 0) || + rewrite(min(x, y), b, min_diff(x, y, this) >= 0))) || rewrite(min(x, c0), b, is_min_value(c0)) || rewrite(min(x, c0), a, is_max_value(c0)) || rewrite(min((x / c0) * c0, x), a, c0 > 0) || diff --git a/src/Simplify_Stmts.cpp b/src/Simplify_Stmts.cpp index c0e942a177ca..870d7f91e44a 100644 --- a/src/Simplify_Stmts.cpp +++ b/src/Simplify_Stmts.cpp @@ -1,3 +1,4 @@ +#include "Simplify.h" #include "Simplify_Internal.h" #include diff --git a/src/Type.h b/src/Type.h index fae4db772562..a56dd62c810d 100644 --- a/src/Type.h +++ b/src/Type.h @@ -6,6 +6,7 @@ #include "Util.h" #include "runtime/HalideRuntime.h" #include +#include #include /** \file @@ -383,6 +384,16 @@ struct Type { return type_lanes; } + /** A cheap hash of the type, for use in the hashes of Expr nodes that + * embed a Type (see Expr.h). Just the bits of type_code, type_bits, and + * type_lanes (which happen to pack into 32 bits), ignoring handle_index_. */ + HALIDE_ALWAYS_INLINE + uint32_t hash() const { + uint32_t result; + memcpy(&result, this, sizeof(result)); + return result; + } + /** Return Type with same number of bits and lanes, but new_code for a type code. */ HALIDE_ALWAYS_INLINE Type with_code(halide_type_code_t new_code) const { diff --git a/test/correctness/simplify.cpp b/test/correctness/simplify.cpp index 981a00e6f0ee..986cdfc41eb1 100644 --- a/test/correctness/simplify.cpp +++ b/test/correctness/simplify.cpp @@ -2377,6 +2377,18 @@ void check_invariant() { } } +void check_with_assumptions(const Expr &a, const Expr &b, const std::vector &assumptions) { + Expr simpler = simplify(a, Scope(), Scope(), assumptions); + if (!equal(simpler, b)) { + std::cerr + << "\nSimplification failure:\n" + << "Input: " << a << "\n" + << "Output: " << simpler << "\n" + << "Expected output: " << b << "\n"; + abort(); + } +} + void check_unreachable() { Var x("x"), y("y"); @@ -2405,6 +2417,136 @@ void check_unreachable() { Evaluate::make(0)); } +void check_facts() { + Expr x = Var("x"), y = Var("y"), z = Var("z"); + + // These rules are for the part of lowering that runs once regions and + // allocation sizes have been read out of the IR, so test them there. + ScopedRegionsInferred regions_inferred; + + // A fact stated in any comparison direction should let the simplifier pick + // the winning side of a max or min. + check_with_assumptions(max(x, y), x, {x > y}); + check_with_assumptions(max(x, y), x, {y < x}); + check_with_assumptions(max(x, y), y, {x < y}); + check_with_assumptions(max(x, y), y, {y > x}); + check_with_assumptions(min(x, y), y, {x > y}); + check_with_assumptions(min(x, y), x, {x < y}); + + // A non-strict fact is enough to pick a side of a max or min, and a strict + // fact implies the non-strict one. + check_with_assumptions(max(x, y), x, {x >= y}); + check_with_assumptions(max(x, y), y, {x <= y}); + check_with_assumptions(min(x, y), x, {x <= y}); + check_with_assumptions(min(x, y), y, {x >= y}); + + // Facts about compound expressions work too. + check_with_assumptions(max(x + z, y * 3), x + z, {x + z > y * 3}); + check_with_assumptions(max(max(x, y), z), z, {max(x, y) < z}); + + // Both branches of an if learn from the condition, in opposite directions. + check(IfThenElse::make(x < y, not_no_op(max(x, y)), not_no_op(max(x, y))), + IfThenElse::make(x < y, not_no_op(y), not_no_op(x))); + + // A fact only applies where it holds. + check(Block::make(not_no_op(max(x, y)), + IfThenElse::make(x < y, not_no_op(max(x, y)))), + Block::make(not_no_op(max(x, y)), + IfThenElse::make(x < y, not_no_op(y)))); + + // A division can cancel a multiplication inside a max or min when we know + // which side wins after the division. + check_with_assumptions(max(x * 8, y) / 8, x, {x >= y / 8}); + check_with_assumptions(max(y, x * 8) / 8, x, {x >= y / 8}); + check_with_assumptions(min(x * 8, y) / 8, x, {x <= y / 8}); + check_with_assumptions(min(y, x * 8) / 8, x, {x <= y / 8}); + + // The direction in which a fact is stated doesn't matter, on either side: + // both the facts and the conditions of can_prove predicates are looked up + // in the same canonical form. + check_with_assumptions(max(x * 8, y) / 8, x, {y / 8 <= x}); + check_with_assumptions(max(x * 8, y) / 8, x, {!(x < y / 8)}); + check_with_assumptions(min(x * 8, y) / 8, x, {y / 8 >= x}); + + // A strict fact settles a non-strict predicate too. + check_with_assumptions(max(x * 8, y) / 8, x, {x > y / 8}); + check_with_assumptions(min(x * 8, y) / 8, x, {x < y / 8}); + + // A difference only means what we take it to mean where the type cannot + // wrap. Given x >= y + 5 over uint8, y = 253 makes y + 5 equal 2, so x = 10 + // satisfies it while sitting far below y: ordering the min from that would + // pick the wrong side. Only the types whose overflow is undefined, and so + // may be assumed not to happen, are eligible. + for (Type t : {UInt(8), Int(8), Int(16), UInt(32)}) { + Expr a = Variable::make(t, "wrap_a"); + Expr b = Variable::make(t, "wrap_b"); + check_with_assumptions(min(a, b), min(a, b), {a >= b + cast(t, 5)}); + check_with_assumptions(max(a, b), max(a, b), {a >= b + cast(t, 5)}); + } + for (Type t : {Int(32), Int(64)}) { + Expr a = Variable::make(t, "wrap_a"); + Expr b = Variable::make(t, "wrap_b"); + check_with_assumptions(min(a, b), b, {a >= b + cast(t, 5)}); + check_with_assumptions(max(a, b), a, {a >= b + cast(t, 5)}); + } + + // A min is at most either of its operands and a max is at least either of + // them, which needs no facts at all. That only bounds the difference on one + // side, but knowing the two are unequal removes the endpoint, and the two + // together settle a comparison that neither settles alone. + check_with_assumptions(max(min(x, y) + 1, x), x, {min(x, y) != x}); + check_with_assumptions(min(max(x, y) - 1, x), x, {max(x, y) != x}); + + // Neither ingredient is enough by itself: without the inequality the + // difference could still be zero, and without the shape there is no bound + // for the inequality to tighten. + check_with_assumptions(max(min(x, y) + 1, x), max(min(x, y) + 1, x), {z < z + 1}); + check_with_assumptions(max(y + 1, x), max(y + 1, x), {y != x}); + + // Deeply nested mins and maxes must not make the work of proving the + // predicates of the rules above blow up. + Expr nest = x; + for (int i = 0; i < 24; i++) { + nest = min(max(nest + i, y - i), z * i); + } + // The result isn't interesting; what matters is that we get one at all. + (void)simplify(nest, Scope(), Scope(), {x < y}); + + // can_prove-based rules (unlike the known_true ones above) recursively + // invoke the simplifier on their own predicate, and that predicate can be + // a freshly built expression rather than a piece of the original IR (e.g. + // min(x, y) - min(z, w) -> y - w, can_prove(x - y == z - w)) constructs a + // brand new subtraction). If the operands are themselves unsimplified + // instances of the same shape, this recurses; the depth limit must bound + // the work rather than let it explode. + Expr deep = min(Var("da"), Var("db")) - min(Var("dc"), Var("dd")); + for (int i = 0; i < 10; i++) { + Expr y = Var("dy" + std::to_string(i)); + Expr z = Var("dz" + std::to_string(i)); + Expr w = Var("dw" + std::to_string(i)); + deep = min(deep, y) - min(z, w); + } + (void)simplify(deep); + + // Constant offsets are peeled off both the facts and the queries, so a fact + // stated about a shifted operand still settles a predicate about the + // unshifted one, in either direction. + check_with_assumptions(max(x, y), y, {x + 1 <= y}); + check_with_assumptions(max(x, y), x, {y <= x + 0}); + check_with_assumptions(max(x + 3, y), y, {x + 4 <= y}); + check_with_assumptions(min(x, y), x, {x + 1 <= y}); + + // But an offset that leaves the order undetermined still doesn't fire. + check_with_assumptions(max(x, y), max(x, y), {x <= y + 1}); + + // Without the fact, the division stays put. + check(max(x * 8, y) / 8, max(x * 8, y) / 8); + + // Facts that don't strictly order the operands don't fire these rules. + check_with_assumptions(max(x, y), max(x, y), {x != y}); + check_with_assumptions(max(x * 8, y) / 8, max(x * 8, y) / 8, {x < y / 8}); +} + int main(int argc, char **argv) { check_invariant(); check_casts(); @@ -2417,6 +2559,7 @@ int main(int argc, char **argv) { check_bitwise(); check_lets(); check_unreachable(); + check_facts(); // Miscellaneous cases that don't fit into one of the categories above. Expr x = Var("x"), y = Var("y");