/* * Copyright (c) Meta Platforms, Inc. and affiliates. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef FOLLY_GEN_BASE_H_ #error This file may only be included from folly/gen/Base.h #endif #include <folly/Function.h> #include <folly/Portability.h> #include <folly/container/F14Map.h> #include <folly/container/F14Set.h> #include <folly/functional/Invoke.h> // inner condition from: // https://github.com/ericniebler/range-v3/blob/0.11.0/include/range/v3/detail/config.hpp#L222 #define FOLLY_DETAIL_GEN_BASE_HAS_RANGEV3 … #if FOLLY_DETAIL_GEN_BASE_HAS_RANGEV3 #include <range/v3/view/filter.hpp> #include <range/v3/view/transform.hpp> #endif // Ignore shadowing warnings within this file, so includers can use -Wshadow. FOLLY_PUSH_WARNING FOLLY_GNU_DISABLE_WARNING(…) namespace folly { namespace gen { /** * ArgumentReference - For determining ideal argument type to receive a value. */ template <class T> struct ArgumentReference : public std::conditional< std::is_reference<T>::value, T, // T& -> T&, T&& -> T&&, const T& -> const T& typename std::conditional< std::is_const<T>::value, T&, // const int -> const int& T&& // int -> int&& >::type> { … }; /** * Group - The output objects from the GroupBy operator */ template <class Key, class Value> class Group : public GenImpl<Value&&, Group<Key, Value>> { … }; namespace detail { // Classes used for the implementation of Sources, Operators, and Sinks /* ******************************* Sources *************************************** */ /* * ReferencedSource - Generate values from an STL-like container using * iterators from .begin() until .end(). Value type defaults to the type of * *container->begin(). For std::vector<int>, this would be int&. Note that the * value here is a reference, so the values in the vector will be passed by * reference to downstream operators. * * This type is primarily used through the 'from' helper method, like: * * string& longestName = from(names) * | maxBy([](string& s) { return s.size() }); */ template <class Container, class Value> class ReferencedSource : public GenImpl<Value, ReferencedSource<Container, Value>> { … }; /** * CopiedSource - For producing values from eagerly from a sequence of values * whose storage is owned by this class. Useful for preparing a generator for * use after a source collection will no longer be available, or for when the * values are specified literally with an initializer list. * * This type is primarily used through the 'fromCopy' function, like: * * auto sourceCopy = fromCopy(makeAVector()); * auto sum = sourceCopy | sum; * auto max = sourceCopy | max; * * Though it is also used for the initializer_list specialization of from(). */ template <class StorageType, class Container> class CopiedSource : public GenImpl<const StorageType&, CopiedSource<StorageType, Container>> { … }; /** * RangeSource - For producing values from a folly::Range. Useful for referring * to a slice of some container. * * This type is primarily used through the 'from' function, like: * * auto rangeSource = from(folly::range(v.begin(), v.end())); * auto sum = rangeSource | sum; * * Reminder: Be careful not to invalidate iterators when using ranges like this. */ template <class Iterator> class RangeSource : public GenImpl< typename Range<Iterator>::reference, RangeSource<Iterator>> { … }; /** * Sequence - For generating values from beginning value, incremented along the * way with the ++ and += operators. Iteration may continue indefinitely. * Value type specified explicitly. * * This type is primarily used through the 'seq' and 'range' function, like: * * int total = seq(1, 10) | sum; * auto indexes = range(0, 10); * auto endless = seq(0); // 0, 1, 2, 3, ... */ template <class Value, class SequenceImpl> class Sequence : public GenImpl<const Value&, Sequence<Value, SequenceImpl>> { … }; /** * Sequence implementations (range, sequence, infinite, with/without step) **/ template <class Value> class RangeImpl { … }; template <class Value, class Distance> class RangeWithStepImpl { … }; template <class Value> class SeqImpl { … }; template <class Value, class Distance> class SeqWithStepImpl { … }; template <class Value> class InfiniteImpl { … }; /** * GenratorBuilder - Helper for GENERTATOR macro. **/ template <class Value> struct GeneratorBuilder { … }; /** * Yield - For producing values from a user-defined generator by way of a * 'yield' function. **/ template <class Value, class Source> class Yield : public GenImpl<Value, Yield<Value, Source>> { … }; template <class Value> class Empty : public GenImpl<Value, Empty<Value>> { … }; template <class Value> class SingleReference : public GenImpl<Value&, SingleReference<Value>> { … }; template <class Value> class SingleCopy : public GenImpl<const Value&, SingleCopy<Value>> { … }; /* ***************************** Operators *************************************** */ /** * Map - For producing a sequence of values by passing each value from a source * collection through a predicate. * * This type is usually used through the 'map' or 'mapped' helper function: * * auto squares = seq(1, 10) | map(square) | as<std::vector>(); */ template <class Predicate> class Map : public Operator<Map<Predicate>> { … }; /** * Filter - For filtering values from a source sequence by a predicate. * * This type is usually used through the 'filter' helper function, like: * * auto nonEmpty = from(strings) * | filter([](const string& str) -> bool { * return !str.empty(); * }); * * Note that if no predicate is provided, the values are casted to bool and * filtered based on that. So if pointers is a vector of pointers, * * auto nonNull = from(pointers) | filter(); * * will give a vector of all the pointers != nullptr. */ template <class Predicate> class Filter : public Operator<Filter<Predicate>> { … }; /** * Until - For producing values from a source until a predicate is satisfied. * * This type is usually used through the 'until' helper function, like: * * auto best = from(sortedItems) * | until([](Item& item) { return item.score > 100; }) * | as<std::vector>(); */ template <class Predicate> class Until : public Operator<Until<Predicate>> { … }; /** * Take - For producing up to N values from a source. * * This type is usually used through the 'take' helper function, like: * * auto best = from(docs) * | orderByDescending(scoreDoc) * | take(10); */ class Take : public Operator<Take> { … }; /** * Visit - For calling a function on each item before passing it down the * pipeline. * * This type is usually used through the 'visit' helper function: * * auto printedValues = seq(1) | visit(debugPrint); * // nothing printed yet * auto results = take(10) | as<std::vector>(); * // results now populated, 10 values printed */ template <class Visitor> class Visit : public Operator<Visit<Visitor>> { … }; /** * Stride - For producing every Nth value from a source. * * This type is usually used through the 'stride' helper function, like: * * auto half = from(samples) * | stride(2); */ class Stride : public Operator<Stride> { … }; /** * Sample - For taking a random sample of N elements from a sequence * (without replacement). */ template <class Random> class Sample : public Operator<Sample<Random>> { … }; /** * Skip - For skipping N items from the beginning of a source generator. * * This type is usually used through the 'skip' helper function, like: * * auto page = from(results) * | skip(pageSize * startPage) * | take(10); */ class Skip : public Operator<Skip> { … }; /** * Order - For ordering a sequence of values from a source by key. * The key is extracted by the given selector functor, and this key is then * compared using the specified comparator. * * This type is usually used through the 'order' helper function, like: * * auto closest = from(places) * | orderBy([](Place& p) { * return -distance(p.location, here); * }) * | take(10); */ template <class Selector, class Comparer> class Order : public Operator<Order<Selector, Comparer>> { … }; /** * GroupBy - Group values by a given key selector, producing a sequence of * groups. * * This type is usually used through the 'groupBy' helper function, like: * * auto bests * = from(places) * | groupBy([](const Place& p) { * return p.city; * }) * | [](Group<std::string, Place>&& g) { * cout << g.key() << ": " << (g | first).description; * }; */ template <class Selector> class GroupBy : public Operator<GroupBy<Selector>> { … }; /** * GroupByAdjacent - Group adjacent values by a given key selector, producing a * sequence of groups. This differs from GroupBy in that only contiguous sets * of values with the same key are considered part of the same group. Unlike * GroupBy, this can be used on infinite sequences. * * This type is usually used through the 'groupByAdjacent' helper function: * * auto tens * = seq(0) * | groupByAdjacent([](int i){ return (i / 10) % 2; }) * * This example results in a list like [ 0:[0-9], 1:[10-19], 0:[20-29], ... ] */ template <class Selector> class GroupByAdjacent : public Operator<GroupByAdjacent<Selector>> { … }; /* * TypeAssertion - For verifying the exact type of the value produced by a * generator. Useful for testing and debugging, and acts as a no-op at runtime. * Pass-through at runtime. Used through the 'assert_type<>()' factory method * like so: * * auto c = from(vector) | assert_type<int&>() | sum; * */ template <class Expected> class TypeAssertion : public Operator<TypeAssertion<Expected>> { … }; /** * Distinct - For filtering duplicates out of a sequence. A selector may be * provided to generate a key to uniquify for each value. * * This type is usually used through the 'distinct' helper function, like: * * auto closest = from(results) * | distinctBy([](Item& i) { * return i.target; * }) * | take(10); */ template <class Selector> class Distinct : public Operator<Distinct<Selector>> { … }; /** * Composer - Helper class for adapting pipelines into functors. Primarily used * for 'mapOp'. */ template <class Operators> class Composer { … }; /** * Batch - For producing fixed-size batches of each value from a source. * * This type is usually used through the 'batch' helper function: * * auto batchSums * = seq(1, 10) * | batch(3) * | map([](const std::vector<int>& batch) { * return from(batch) | sum; * }) * | as<vector>(); */ class Batch : public Operator<Batch> { … }; /** * Window - For overlapping the lifetimes of pipeline values, especially with * Futures. * * This type is usually used through the 'window' helper function: * * auto responses * = byLine(STDIN) * | map(makeRequestFuture) * | window(1000) * | map(waitFuture) * | as<vector>(); */ class Window : public Operator<Window> { … }; /** * Concat - For flattening generators of generators. * * This type is usually used through the 'concat' static value, like: * * auto edges = * from(nodes) * | map([](Node& x) { * return from(x.neighbors) * | map([&](Node& y) { * return Edge(x, y); * }); * }) * | concat * | as<std::set>(); */ class Concat : public Operator<Concat> { … }; /** * RangeConcat - For flattening generators of iterables. * * This type is usually used through the 'rconcat' static value, like: * * map<int, vector<int>> adjacency; * auto sinks = * from(adjacency) * | get<1>() * | rconcat() * | as<std::set>(); */ class RangeConcat : public Operator<RangeConcat> { … }; /** * GuardImpl - For handling exceptions from downstream computation. Requires the * type of exception to catch, and handler function to invoke in the event of * the exception. Note that the handler may: * 1) return true to continue processing the sequence * 2) return false to end the sequence immediately * 3) throw, to pass the exception to the next catch * The handler must match the signature 'bool(Exception&, Value)'. * * This type is used through the `guard` helper, like so: * * auto indexes * = byLine(STDIN_FILENO) * | guard<std::runtime_error>([](std::runtime_error& e, * StringPiece sp) { * LOG(ERROR) << sp << ": " << e.str(); * return true; // continue processing subsequent lines * }) * | eachTo<int>() * | as<vector>(); * * KNOWN ISSUE: This only guards pipelines through operators which do not * retain resulting values. Exceptions thrown after operators like pmap, order, * batch, cannot be caught from here. **/ template <class Exception, class ErrorHandler> class GuardImpl : public Operator<GuardImpl<Exception, ErrorHandler>> { … }; /** * Dereference - For dereferencing a sequence of pointers while filtering out * null pointers. * * This type is usually used through the 'dereference' static value, like: * * auto refs = from(ptrs) | dereference; */ class Dereference : public Operator<Dereference> { … }; /** * Indirect - For producing a sequence of the addresses of the values in the * input. * * This type is usually used through the 'indirect' static value, like: * * auto ptrs = from(refs) | indirect; */ class Indirect : public Operator<Indirect> { … }; /** * Cycle - For repeating a sequence forever. * * This type is usually used through the 'cycle' static value, like: * * auto tests * = from(samples) * | cycle * | take(100); * * or in the finite case: * * auto thrice = g | cycle(3); */ template <bool forever> class Cycle : public Operator<Cycle<forever>> { … }; /* ******************************* Sinks ***************************************** */ /** * FoldLeft - Left-associative functional fold. For producing an aggregate value * from a seed and a folder function. Useful for custom aggregators on a * sequence. * * This type is primarily used through the 'foldl' helper method, like: * * double movingAverage = from(values) * | foldl(0.0, [](double avg, double sample) { * return sample * 0.1 + avg * 0.9; * }); */ template <class Seed, class Fold> class FoldLeft : public Operator<FoldLeft<Seed, Fold>> { … }; /** * First - For finding the first value in a sequence. * * This type is primarily used through the 'first' static value, like: * * int firstThreeDigitPrime = seq(100) | filter(isPrime) | first; */ class First : public Operator<First> { … }; /** * IsEmpty - a helper class for isEmpty and notEmpty * * Essentially returns 'result' if the source is empty. Note that this cannot be * called on an infinite source, because then there is only one possible return * value. * * * Used primarily through 'isEmpty' and 'notEmpty' static values * * bool hasPrimes = g | filter(prime) | notEmpty; * bool lacksEvens = g | filter(even) | isEmpty; * * Also used in the implementation of 'any' and 'all' */ template <bool emptyResult> class IsEmpty : public Operator<IsEmpty<emptyResult>> { … }; /** * Reduce - Functional reduce, for recursively combining values from a source * using a reducer function until there is only one item left. Useful for * combining values when an empty sequence doesn't make sense. * * This type is primarily used through the 'reduce' helper method, like: * * sring longest = from(names) * | reduce([](string&& best, string& current) { * return best.size() >= current.size() ? best : current; * }); */ template <class Reducer> class Reduce : public Operator<Reduce<Reducer>> { … }; /** * Count - for simply counting the items in a collection. * * This type is usually used through its singleton, 'count': * * auto shortPrimes = seq(1, 100) | filter(isPrime) | count; */ class Count : public Operator<Count> { … }; /** * Sum - For simply summing up all the values from a source. * * This type is usually used through its singleton, 'sum': * * auto gaussSum = seq(1, 100) | sum; */ class Sum : public Operator<Sum> { … }; /** * Contains - For testing whether a value matching the given value is contained * in a sequence. * * This type should be used through the 'contains' helper method, like: * * bool contained = seq(1, 10) | map(square) | contains(49); */ template <class Needle> class Contains : public Operator<Contains<Needle>> { … }; /** * Min - For a value which minimizes a key, where the key is determined by a * given selector, and compared by given comparer. * * This type is usually used through the singletone 'min' or through the helper * functions 'minBy' and 'maxBy'. * * auto oldest = from(people) * | minBy([](Person& p) { * return p.dateOfBirth; * }); */ template <class Selector, class Comparer> class Min : public Operator<Min<Selector, Comparer>> { … }; /** * Append - For collecting values from a source into a given output container * by appending. * * This type is usually used through the helper function 'appendTo', like: * * vector<int64_t> ids; * from(results) | map([](Person& p) { return p.id }) * | appendTo(ids); */ template <class Collection> class Append : public Operator<Append<Collection>> { … }; /** * Collect - For collecting values from a source in a collection of the desired * type. * * This type is usually used through the helper function 'as', like: * * std::string upper = from(stringPiece) * | map(&toupper) * | as<std::string>(); */ template <class Collection> class Collect : public Operator<Collect<Collection>> { … }; /** * CollectTemplate - For collecting values from a source in a collection * constructed using the specified template type. Given the type of values * produced by the given generator, the collection type will be: * Container<Value, Allocator<Value>> * * The allocator defaults to std::allocator, so this may be used for the STL * containers by simply using operators like 'as<set>', 'as<deque>', * 'as<vector>'. 'as', here is the helper method which is the usual means of * constructing this operator. * * Example: * * set<string> uniqueNames = from(names) | as<set>(); */ template < template <class, class> class Container, template <class> class Allocator> class CollectTemplate : public Operator<CollectTemplate<Container, Allocator>> { … }; /** * UnwrapOr - For unwrapping folly::Optional values, or providing the given * fallback value. Usually used through the 'unwrapOr' helper like so: * * auto best = from(scores) | max | unwrapOr(-1); * * Note that the fallback value needn't match the value in the Optional it is * unwrapping. If mis-matched types are supported, the common type of the two is * returned by value. If the types match, a reference (T&& > T& > const T&) is * returned. */ template <class T> class UnwrapOr { … }; template <class T> T&& operator|(Optional<T>&& opt, UnwrapOr<T>&& fallback) { … } template <class T> T& operator|(Optional<T>& opt, UnwrapOr<T>& fallback) { … } template <class T> const T& operator|(const Optional<T>& opt, const UnwrapOr<T>& fallback) { … } // Mixed type unwrapping always returns values, moving where possible template < class T, class U, class R = typename std::enable_if< !std::is_same<T, U>::value, typename std::common_type<T, U>::type>::type> R operator|(Optional<T>&& opt, UnwrapOr<U>&& fallback) { … } template < class T, class U, class R = typename std::enable_if< !std::is_same<T, U>::value, typename std::common_type<T, U>::type>::type> R operator|(const Optional<T>& opt, UnwrapOr<U>&& fallback) { … } template < class T, class U, class R = typename std::enable_if< !std::is_same<T, U>::value, typename std::common_type<T, U>::type>::type> R operator|(Optional<T>&& opt, const UnwrapOr<U>& fallback) { … } template < class T, class U, class R = typename std::enable_if< !std::is_same<T, U>::value, typename std::common_type<T, U>::type>::type> R operator|(const Optional<T>& opt, const UnwrapOr<U>& fallback) { … } /** * Unwrap - For unwrapping folly::Optional values in a folly::gen style. Usually * used through the 'unwrap' instace like so: * * auto best = from(scores) | max | unwrap; // may throw */ class Unwrap { … }; template <class T> T&& operator|(Optional<T>&& opt, const Unwrap&) { … } template <class T> T& operator|(Optional<T>& opt, const Unwrap&) { … } template <class T> const T& operator|(const Optional<T>& opt, const Unwrap&) { … } class ToVirtualGen : public Operator<ToVirtualGen> { … }; #if FOLLY_DETAIL_GEN_BASE_HAS_RANGEV3 template <class RangeV3, class Value> class RangeV3Source : public gen::GenImpl<Value, RangeV3Source<RangeV3, Value>> { mutable RangeV3 r_; // mutable since some ranges are not const-iteratable public: explicit RangeV3Source(RangeV3 const& r) : r_(r) {} template <class Body> void foreach(Body&& body) const { for (auto const& value : r_) { body(value); } } template <class Handler> bool apply(Handler&& handler) const { for (auto const& value : r_) { if (!handler(value)) { return false; } } return true; } static constexpr bool infinite = false; }; template <class RangeV3, class Value> class RangeV3CopySource : public gen::GenImpl<Value, RangeV3CopySource<RangeV3, Value>> { mutable RangeV3 r_; // mutable since some ranges are not const-iteratable public: explicit RangeV3CopySource(RangeV3&& r) : r_(std::move(r)) {} template <class Body> void foreach(Body&& body) const { for (auto const& value : r_) { body(value); } } template <class Handler> bool apply(Handler&& handler) const { for (auto const& value : r_) { if (!handler(value)) { return false; } } return true; } static constexpr bool infinite = false; }; struct from_container_fn { template <typename Container> friend auto operator|(Container&& c, from_container_fn) { return gen::from(std::forward<Container>(c)); } }; struct from_rangev3_fn { template <typename Range> friend auto operator|(Range&& r, from_rangev3_fn) { using DecayedRange = std::decay_t<Range>; using DecayedValue = std::decay_t<decltype(*r.begin())>; return RangeV3Source<DecayedRange, DecayedValue>(r); } }; struct from_rangev3_copy_fn { template <typename Range> friend auto operator|(Range&& r, from_rangev3_copy_fn) { using RangeDecay = std::decay_t<Range>; using Value = std::decay_t<decltype(*r.begin())>; return RangeV3CopySource<RangeDecay, Value>(std::move(r)); } }; #endif // FOLLY_DETAIL_GEN_BASE_HAS_RANGEV3 } // namespace detail #if FOLLY_DETAIL_GEN_BASE_HAS_RANGEV3 /* ****************************************************************************** * Pipe fittings between a container/range-v3 and a folly::gen. * Example: vec | gen::from_container | folly::gen::filter(...); * Example: vec | ranges::views::filter(...) | gen::from_rangev3 | gen::xxx; ****************************************************************************** */ constexpr detail::from_container_fn from_container; constexpr detail::from_rangev3_fn from_rangev3; constexpr detail::from_rangev3_copy_fn from_rangev3_copy; template <typename Range> auto from_rangev3_call(Range&& r) { using Value = std::decay_t<decltype(*r.begin())>; return detail::RangeV3Source<Range, Value>(r); } // it is safe to pipe an rvalue into a range-v3 view if the rest of the pipeline // will finish its traversal within the current full-expr, a condition provided // by folly::gen. template <typename Range> auto rangev3_will_be_consumed(Range&& r) { // intentionally use `r` instead of `std::forward<Range>(r)`; see above. // range-v3 ranges copy in O(1) so it is appropriate. return ::ranges::views::all(r); } #endif // FOLLY_DETAIL_GEN_BASE_HAS_RANGEV3 /** * VirtualGen<T> - For wrapping template types in simple polymorphic wrapper. **/ template <class Value, class Self> class VirtualGenBase : public GenImpl<Value, Self> { … }; template <class Value> class VirtualGen : public VirtualGenBase<Value, VirtualGen<Value>> { … }; template <class Value> class VirtualGenMoveOnly : public VirtualGenBase<Value, VirtualGenMoveOnly<Value>> { … }; /** * non-template operators, statically defined to avoid the need for anything but * the header. */ constexpr detail::Sum sum{ … }; constexpr detail::Count count{ … }; constexpr detail::First first{ … }; constexpr detail::IsEmpty<true> isEmpty{ … }; constexpr detail::IsEmpty<false> notEmpty{ … }; constexpr detail::Min<Identity, Less> min{ … }; constexpr detail::Min<Identity, Greater> max{ … }; constexpr detail::Order<Identity> order{ … }; constexpr detail::Distinct<Identity> distinct{ … }; constexpr detail::Map<Move> move{ … }; constexpr detail::Concat concat{ … }; constexpr detail::RangeConcat rconcat{ … }; constexpr detail::Cycle<true> cycle{ … }; constexpr detail::Dereference dereference{ … }; constexpr detail::Indirect indirect{ … }; constexpr detail::Unwrap unwrap{ … }; constexpr detail::ToVirtualGen virtualize{ … }; template <class Number> inline detail::Take take(Number count) { … } inline detail::Stride stride(size_t s) { … } template <class Random = std::default_random_engine> inline detail::Sample<Random> sample(size_t count, Random rng = Random()) { … } inline detail::Skip skip(size_t count) { … } inline detail::Batch batch(size_t batchSize) { … } inline detail::Window window(size_t windowSize) { … } } // namespace gen } // namespace folly FOLLY_POP_WARNING