From d8bc53b7abc938f98f0bf8b3315438838bf8fa92 Mon Sep 17 00:00:00 2001 From: Harendra Kumar Date: Fri, 20 Aug 2021 01:05:06 +0530 Subject: [PATCH 1/6] Remove dependency on the IsStream.Enumeration module --- src/Streamly/Internal/Data/Fold.hs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Streamly/Internal/Data/Fold.hs b/src/Streamly/Internal/Data/Fold.hs index 2b59f79692..dddaf4353f 100644 --- a/src/Streamly/Internal/Data/Fold.hs +++ b/src/Streamly/Internal/Data/Fold.hs @@ -256,7 +256,7 @@ import Streamly.Internal.Data.Stream.Serial (SerialT) import qualified Data.Map.Strict as Map import qualified Streamly.Internal.Data.Pipe.Type as Pipe -import qualified Streamly.Internal.Data.Stream.IsStream.Enumeration as Stream +-- import qualified Streamly.Internal.Data.Stream.IsStream.Enumeration as Stream import qualified Streamly.Internal.Data.Stream.StreamK as K import qualified Prelude @@ -1617,8 +1617,9 @@ zip = zipWithM (curry return) -- -- /Unimplemented/ {-# INLINE indexed #-} -indexed :: forall m a b. Monad m => Fold m (Int, a) b -> Fold m a b -indexed = zip (Stream.enumerateFrom 0 :: SerialT m Int) +indexed :: -- forall m a b. Monad m => + Fold m (Int, a) b -> Fold m a b +indexed = undefined -- zip (Stream.enumerateFrom 0 :: SerialT m Int) -- | Change the predicate function of a Fold from @a -> b@ to accept an -- additional state input @(s, a) -> b@. Convenient to filter with an From 70e7f94691d93073f8e22d47ec0f0fe05ed50c84 Mon Sep 17 00:00:00 2001 From: Harendra Kumar Date: Sat, 4 Sep 2021 11:17:27 +0530 Subject: [PATCH 2/6] Move IsStream definition out of low level modules The IsStream type class contains MonadAsync type in the consM method. This refactor allows lifting MonadAsync, therefore, dependency on monad-control out of the low level modules. --- .hlint.ignore | 5 + .../Streamly/Benchmark/Data/Stream/StreamK.hs | 43 +- benchmark/Streamly/Benchmark/Prelude/Async.hs | 4 +- .../Streamly/Benchmark/Prelude/Parallel.hs | 3 +- .../Prelude/Serial/Transformation3.hs | 1 + benchmark/lib/Streamly/Benchmark/Prelude.hs | 4 +- hie.yaml | 8 + src/Streamly.hs | 25 +- src/Streamly/Internal/Data/Array.hs | 17 +- src/Streamly/Internal/Data/Array/Foreign.hs | 15 +- .../Internal/Data/Array/Foreign/Mut/Type.hs | 4 +- .../Internal/Data/Array/Foreign/Type.hs | 13 +- .../Internal/Data/Array/Prim/TypesInclude.hs | 6 +- .../Internal/Data/Array/PrimInclude.hs | 29 +- .../Internal/Data/Array/Stream/Foreign.hs | 35 +- .../Internal/Data/Array/Stream/Mut/Foreign.hs | 17 +- src/Streamly/Internal/Data/Fold.hs | 10 +- src/Streamly/Internal/Data/Fold/Async.hs | 120 +++ src/Streamly/Internal/Data/Fold/Type.hs | 95 +- src/Streamly/Internal/Data/IORef/Prim.hs | 10 - src/Streamly/Internal/Data/List.hs | 39 +- src/Streamly/Internal/Data/SmallArray.hs | 15 +- src/Streamly/Internal/Data/Stream/Ahead.hs | 149 +-- src/Streamly/Internal/Data/Stream/Async.hs | 324 ++----- .../Internal/Data/Stream/Instances.hs | 24 +- src/Streamly/Internal/Data/Stream/IsStream.hs | 6 +- .../Data/Stream/IsStream/Combinators.hs | 7 +- .../Internal/Data/Stream/IsStream/Common.hs | 143 ++- .../Data/Stream/IsStream/Eliminate.hs | 74 +- .../Data/Stream/IsStream/Enumeration.hs | 33 +- .../Data/Stream/IsStream/Exception.hs | 30 +- .../Internal/Data/Stream/IsStream/Expand.hs | 458 ++++++++-- .../Internal/Data/Stream/IsStream/Generate.hs | 155 +++- .../Internal/Data/Stream/IsStream/Lift.hs | 5 +- .../Internal/Data/Stream/IsStream/Reduce.hs | 68 +- .../Internal/Data/Stream/IsStream/Top.hs | 30 +- .../Data/Stream/IsStream/Transform.hs | 135 ++- .../Internal/Data/Stream/IsStream/Type.hs | 698 ++++++++++++++ .../Internal/Data/Stream/IsStream/Types.hs | 74 -- src/Streamly/Internal/Data/Stream/Parallel.hs | 231 ++--- src/Streamly/Internal/Data/Stream/Prelude.hs | 321 +------ .../Internal/Data/Stream/PreludeCommon.hs | 139 +++ .../Internal/Data/Stream/SVar/Eliminate.hs | 13 +- .../Internal/Data/Stream/SVar/Generate.hs | 20 +- src/Streamly/Internal/Data/Stream/Serial.hs | 199 ++-- .../Internal/Data/Stream/StreamD/Eliminate.hs | 13 - .../Internal/Data/Stream/StreamD/Generate.hs | 2 - .../Internal/Data/Stream/StreamD/Transform.hs | 16 +- .../Internal/Data/Stream/StreamD/Type.hs | 34 +- src/Streamly/Internal/Data/Stream/StreamK.hs | 425 ++------- .../Internal/Data/Stream/StreamK/Type.hs | 850 ++++++++++-------- src/Streamly/Internal/Data/Stream/Zip.hs | 163 +--- src/Streamly/Internal/Data/Unfold.hs | 7 +- src/Streamly/Internal/FileSystem/Dir.hs | 2 +- src/Streamly/Internal/FileSystem/FD.hs | 9 +- src/Streamly/Internal/FileSystem/File.hs | 4 +- src/Streamly/Internal/FileSystem/Handle.hs | 15 +- src/Streamly/Internal/Network/Inet/TCP.hs | 6 +- src/Streamly/Internal/Network/Socket.hs | 23 +- src/Streamly/Internal/Unicode/Char.hs | 9 +- src/Streamly/Internal/Unicode/Stream.hs | 35 +- src/Streamly/Prelude.hs | 1 + streamly.cabal | 16 +- test/Streamly/Test/Prelude/WSerial.hs | 3 +- 64 files changed, 3035 insertions(+), 2452 deletions(-) create mode 100644 src/Streamly/Internal/Data/Fold/Async.hs create mode 100644 src/Streamly/Internal/Data/Stream/IsStream/Type.hs delete mode 100644 src/Streamly/Internal/Data/Stream/IsStream/Types.hs create mode 100644 src/Streamly/Internal/Data/Stream/PreludeCommon.hs diff --git a/.hlint.ignore b/.hlint.ignore index 414028987b..ed7e5dcc0f 100644 --- a/.hlint.ignore +++ b/.hlint.ignore @@ -1,3 +1,5 @@ +src/Streamly/Internal/Data/Stream/Prelude.hs +src/Streamly/Internal/Data/Stream/PreludeCommon.hs src/Streamly/Internal/Data/Stream/Serial.hs src/Streamly/Internal/Data/Stream/Zip.hs src/Streamly/Internal/Data/Stream/StreamK/Type.hs @@ -6,7 +8,10 @@ src/Streamly/Internal/Data/Pipe/Type.hs src/Streamly/Internal/Data/SmallArray/Type.hs src/Streamly/Internal/Unicode/Stream.hs src/Streamly/Internal/Data/Array/Prim/Type.hs +src/Streamly/Internal/Data/Array/Prim/Mut/Type.hs src/Streamly/Internal/Data/Array/Prim/MutTypesInclude.hs +src/Streamly/Internal/Data/Array/Prim/Pinned/Mut/Type.hs +src/Streamly/Internal/Data/Array/Prim/Pinned/Type.hs src/Streamly/Internal/Data/Array/Prim/TypesInclude.hs test/Streamly/Test/Common/Array.hs test/Streamly/Test/Data/Array.hs diff --git a/benchmark/Streamly/Benchmark/Data/Stream/StreamK.hs b/benchmark/Streamly/Benchmark/Data/Stream/StreamK.hs index fc609b1156..866b2b451c 100644 --- a/benchmark/Streamly/Benchmark/Data/Stream/StreamK.hs +++ b/benchmark/Streamly/Benchmark/Data/Stream/StreamK.hs @@ -32,6 +32,7 @@ import qualified Prelude as P import qualified Data.List as List import qualified Streamly.Internal.Control.Concurrent as S +import qualified Streamly.Internal.Data.Stream.StreamK.Type as S import qualified Streamly.Internal.Data.Stream.StreamK as S import Gauge (bench, nfIO, bgroup, Benchmark, defaultMain) @@ -127,7 +128,7 @@ unfoldr streamLen n = S.unfoldr step n {-# INLINE unfoldrM #-} unfoldrM :: S.MonadAsync m => Int -> Int -> Stream m Int -unfoldrM streamLen n = S.unfoldrM step n +unfoldrM streamLen n = S.unfoldrMWith S.consM step n where step cnt = if cnt > n + streamLen @@ -148,7 +149,7 @@ replicate = S.replicate {-# INLINE replicateM #-} replicateM :: S.MonadAsync m => Int -> Int -> Stream m Int -replicateM streamLen = S.replicateM streamLen . return +replicateM streamLen = S.replicateMWith S.consM streamLen . return {-# INLINE iterate #-} iterate :: Int -> Int -> Stream m Int @@ -168,8 +169,11 @@ fromFoldableM streamLen n = Prelude.foldr S.consM S.nil (Prelude.fmap return [n..n+streamLen]) {-# INLINABLE concatMapFoldableWith #-} -concatMapFoldableWith :: (S.IsStream t, Foldable f) - => (t m b -> t m b -> t m b) -> (a -> t m b) -> f a -> t m b +concatMapFoldableWith :: Foldable f + => (Stream m b -> Stream m b -> Stream m b) + -> (a -> Stream m b) + -> f a + -> Stream m b concatMapFoldableWith f g = Prelude.foldr (f . g) S.nil {-# INLINE concatMapFoldableSerial #-} @@ -202,13 +206,13 @@ uncons s = do Just (_, t) -> uncons t {-# INLINE init #-} -init :: (Monad m, S.IsStream t) => t m a -> m () +init :: Monad m => Stream m a -> m () init s = do t <- S.init s P.mapM_ S.drain t {-# INLINE tail #-} -tail :: (Monad m, S.IsStream t) => t m a -> m () +tail :: Monad m => Stream m a -> m () tail s = S.tail s >>= P.mapM_ tail {-# INLINE nullTail #-} @@ -265,7 +269,7 @@ fmapK n = composeN n $ P.fmap (+ 1) {-# INLINE mapM #-} mapM :: S.MonadAsync m => Int -> Stream m Int -> m () -mapM n = composeN n $ S.mapM return +mapM n = composeN n $ S.mapMWith S.consM return {-# INLINE mapMSerial #-} mapMSerial :: S.MonadAsync m => Int -> Stream m Int -> m () @@ -339,7 +343,8 @@ iterateSource iterStreamLen g i n = f i (unfoldrM iterStreamLen n) -- this is quadratic {-# INLINE iterateScan #-} iterateScan :: S.MonadAsync m => Int -> Int -> Int -> Stream m Int -iterateScan iterStreamLen maxIters = iterateSource iterStreamLen (S.scanl' (+) 0) (maxIters `div` 10) +iterateScan iterStreamLen maxIters = + iterateSource iterStreamLen (S.scanl' (+) 0) (maxIters `div` 10) -- this is quadratic {-# INLINE iterateDropWhileFalse #-} @@ -349,23 +354,27 @@ iterateDropWhileFalse streamLen iterStreamLen maxIters = {-# INLINE iterateMapM #-} iterateMapM :: S.MonadAsync m => Int -> Int -> Int -> Stream m Int -iterateMapM iterStreamLen maxIters = iterateSource iterStreamLen (S.mapM return) maxIters +iterateMapM iterStreamLen = + iterateSource iterStreamLen (S.mapMWith S.consM return) {-# INLINE iterateFilterEven #-} iterateFilterEven :: S.MonadAsync m => Int -> Int -> Int -> Stream m Int -iterateFilterEven iterStreamLen maxIters = iterateSource iterStreamLen (S.filter even) maxIters +iterateFilterEven iterStreamLen = iterateSource iterStreamLen (S.filter even) {-# INLINE iterateTakeAll #-} iterateTakeAll :: S.MonadAsync m => Int -> Int -> Int -> Int -> Stream m Int -iterateTakeAll streamLen iterStreamLen maxIters = iterateSource iterStreamLen (S.take streamLen) maxIters +iterateTakeAll streamLen iterStreamLen = + iterateSource iterStreamLen (S.take streamLen) {-# INLINE iterateDropOne #-} iterateDropOne :: S.MonadAsync m => Int -> Int -> Int -> Stream m Int -iterateDropOne iterStreamLen maxIters = iterateSource iterStreamLen (S.drop 1) maxIters +iterateDropOne iterStreamLen = iterateSource iterStreamLen (S.drop 1) {-# INLINE iterateDropWhileTrue #-} -iterateDropWhileTrue :: S.MonadAsync m => Int -> Int -> Int -> Int -> Stream m Int -iterateDropWhileTrue streamLen iterStreamLen maxIters = iterateSource iterStreamLen (S.dropWhile (<= streamLen)) maxIters +iterateDropWhileTrue :: S.MonadAsync m => + Int -> Int -> Int -> Int -> Stream m Int +iterateDropWhileTrue streamLen iterStreamLen = + iterateSource iterStreamLen (S.dropWhile (<= streamLen)) ------------------------------------------------------------------------------- -- Zipping @@ -471,7 +480,7 @@ sourceConcatMapId val n = {-# INLINE concatMapBySerial #-} concatMapBySerial :: Int -> Int -> Int -> IO () concatMapBySerial outer inner n = - S.drain $ S.concatMapBy S.serial + S.drain $ S.concatMapWith S.serial (unfoldrM inner) (unfoldrM outer n) @@ -681,7 +690,9 @@ o_1_space_concat streamLen = -- This is for comparison with concatMapFoldableWith , benchIOSrc1 "concatMapWithId (n of 1) (fromFoldable)" - (S.drain . S.concatMapBy S.serial id . sourceConcatMapId streamLen) + (S.drain + . S.concatMapWith S.serial id + . sourceConcatMapId streamLen) , benchIOSrc1 "concatMapBy serial (n of 1)" (concatMapBySerial streamLen 1) diff --git a/benchmark/Streamly/Benchmark/Prelude/Async.hs b/benchmark/Streamly/Benchmark/Prelude/Async.hs index 34cf79ea1b..09a862eea1 100644 --- a/benchmark/Streamly/Benchmark/Prelude/Async.hs +++ b/benchmark/Streamly/Benchmark/Prelude/Async.hs @@ -9,7 +9,7 @@ import Prelude hiding (mapM) import Streamly.Prelude (fromAsync, async, maxBuffer, maxThreads, fromSerial) import qualified Streamly.Prelude as S -import qualified Streamly.Internal.Data.Stream.StreamK.Type as Internal +import qualified Streamly.Internal.Data.Stream.IsStream.Transform as Transform import Streamly.Benchmark.Common import Streamly.Benchmark.Prelude @@ -48,7 +48,7 @@ foldrSShared :: Int -> Int -> IO () foldrSShared count n = S.drain $ fromAsync - $ Internal.foldrSShared (\x xs -> S.consM (return x) xs) S.nil + $ Transform.foldrSShared (\x xs -> S.consM (return x) xs) S.nil $ fromSerial $ sourceUnfoldrM count n diff --git a/benchmark/Streamly/Benchmark/Prelude/Parallel.hs b/benchmark/Streamly/Benchmark/Prelude/Parallel.hs index bea3025abe..fd950bfa90 100644 --- a/benchmark/Streamly/Benchmark/Prelude/Parallel.hs +++ b/benchmark/Streamly/Benchmark/Prelude/Parallel.hs @@ -15,7 +15,6 @@ import Streamly.Prelude import qualified Streamly.Prelude as S import qualified Streamly.Internal.Data.Fold as FL -import qualified Streamly.Internal.Data.Stream.Parallel as Par import qualified Streamly.Internal.Data.Stream.IsStream as Internal import Streamly.Benchmark.Common @@ -72,7 +71,7 @@ parAppSum src = (S.sum S.|$. src) >>= \x -> seq x (return ()) {-# INLINE tapAsyncS #-} tapAsyncS :: S.MonadAsync m => Int -> SerialT m Int -> m () -tapAsyncS n = composeN n $ Par.tapAsync S.sum +tapAsyncS n = composeN n $ Internal.tapAsyncK S.sum {-# INLINE tapAsync #-} tapAsync :: S.MonadAsync m => Int -> SerialT m Int -> m () diff --git a/benchmark/Streamly/Benchmark/Prelude/Serial/Transformation3.hs b/benchmark/Streamly/Benchmark/Prelude/Serial/Transformation3.hs index 3f97c07163..3dd660ef5b 100644 --- a/benchmark/Streamly/Benchmark/Prelude/Serial/Transformation3.hs +++ b/benchmark/Streamly/Benchmark/Prelude/Serial/Transformation3.hs @@ -84,6 +84,7 @@ iterateStateT n = do else return x {-# INLINE iterateState #-} +{-# SPECIALIZE iterateState :: Int -> SerialT (StateT Int IO) Int #-} iterateState :: (S.MonadAsync m, MonadState Int m) => Int diff --git a/benchmark/lib/Streamly/Benchmark/Prelude.hs b/benchmark/lib/Streamly/Benchmark/Prelude.hs index ba0e0d7a3a..39255c980e 100644 --- a/benchmark/lib/Streamly/Benchmark/Prelude.hs +++ b/benchmark/lib/Streamly/Benchmark/Prelude.hs @@ -26,6 +26,7 @@ import qualified Data.Foldable as F import qualified Data.List as List import qualified Streamly.Prelude as S import qualified Streamly.Internal.Data.Stream.IsStream as Internal +import qualified Streamly.Internal.Data.Stream.IsStream.Type as IsStream import qualified Streamly.Internal.Data.Pipe as Pipe import qualified Streamly.Internal.Data.Stream.Serial as Serial @@ -89,7 +90,8 @@ sourceUnfoldrM count start = S.unfoldrM step start {-# INLINE sourceUnfoldrMSerial #-} sourceUnfoldrMSerial :: (S.IsStream t, Monad m) => Int -> Int -> t m Int -sourceUnfoldrMSerial count start = Serial.unfoldrM step start +sourceUnfoldrMSerial count start = + IsStream.fromSerial $ Serial.unfoldrM step start where step cnt = if cnt > start + count diff --git a/hie.yaml b/hie.yaml index ea8a1909f0..eb5fe6fc5c 100644 --- a/hie.yaml +++ b/hie.yaml @@ -14,10 +14,18 @@ cradle: component: "bench:Data.Parser.ParserD" - path: "./benchmark/Streamly/Benchmark/Data/Parser/ParserK.hs" component: "bench:Data.Parser.ParserK" + - path: "./benchmark/Streamly/Benchmark/Data/Stream/StreamK.hs" + component: "bench:Data.Stream.StreamK" - path: "./benchmark/Streamly/Benchmark/FileSystem/Handle/Read.hs" component: "bench:FileSystem.Handle" - path: "./benchmark/Streamly/Benchmark/FileSystem/Handle/ReadWrite.hs" component: "bench:FileSystem.Handle" + - path: "./benchmark/Streamly/Benchmark/Prelude/Ahead.hs" + component: "bench:Prelude.Ahead" + - path: "./benchmark/Streamly/Benchmark/Prelude/Async.hs" + component: "bench:Prelude.Async" + - path: "./benchmark/Streamly/Benchmark/Prelude/Parallel.hs" + component: "bench:Prelude.Parallel" - path: "./benchmark/Streamly/Benchmark/Prelude/Serial.hs" component: "bench:Prelude.Serial" - path: "./benchmark/Streamly/Benchmark/Prelude/Serial/Elimination.hs" diff --git a/src/Streamly.hs b/src/Streamly.hs index f98507d8c9..d5aee9edc0 100644 --- a/src/Streamly.hs +++ b/src/Streamly.hs @@ -213,17 +213,16 @@ import Data.Semigroup (Semigroup(..)) import Streamly.Internal.Control.Concurrent (MonadAsync) import Streamly.Internal.Data.SVar (Rate(..)) import Streamly.Internal.Data.Stream.Ahead -import Streamly.Internal.Data.Stream.Async hiding (mkAsync) +import Streamly.Internal.Data.Stream.Async import Streamly.Internal.Data.Stream.IsStream.Combinators -import Streamly.Internal.Data.Stream.Parallel -import Streamly.Internal.Data.Stream.Serial -import Streamly.Internal.Data.Stream.StreamK hiding (serial) +import Streamly.Internal.Data.Stream.IsStream.Expand +import Streamly.Internal.Data.Stream.IsStream.Type +import Streamly.Internal.Data.Stream.Serial (StreamT, InterleavedT) import Streamly.Internal.Data.Stream.Zip import qualified Streamly.Prelude as P import qualified Streamly.Internal.Data.Stream.IsStream as IP -import qualified Streamly.Internal.Data.Stream.StreamK as K -import qualified Streamly.Internal.Data.Stream.Async as Async +import qualified Streamly.Internal.Data.Stream.IsStream.Transform as Transform -- XXX provide good succinct examples of pipelining, merging, splitting etc. -- below. @@ -364,7 +363,7 @@ import qualified Streamly.Internal.Data.Stream.Async as Async -- @since 0.1.0 {-# DEPRECATED runStreaming "Please use runStream instead." #-} runStreaming :: (Monad m, IsStream t) => t m a -> m () -runStreaming = P.drain . K.adapt +runStreaming = P.drain . adapt -- | Same as @runStream@. -- @@ -384,35 +383,35 @@ runStream = P.drain -- @since 0.1.0 {-# DEPRECATED runInterleavedT "Please use 'drain . interleaving' instead." #-} runInterleavedT :: Monad m => WSerialT m a -> m () -runInterleavedT = P.drain . K.adapt +runInterleavedT = P.drain . adapt -- | Same as @drain . fromParallel@. -- -- @since 0.1.0 {-# DEPRECATED runParallelT "Please use 'drain . fromParallel' instead." #-} runParallelT :: Monad m => ParallelT m a -> m () -runParallelT = P.drain . K.adapt +runParallelT = P.drain . adapt -- | Same as @drain . fromAsync@. -- -- @since 0.1.0 {-# DEPRECATED runAsyncT "Please use 'drain . fromAsync' instead." #-} runAsyncT :: Monad m => AsyncT m a -> m () -runAsyncT = P.drain . K.adapt +runAsyncT = P.drain . adapt -- | Same as @drain . zipping@. -- -- @since 0.1.0 {-# DEPRECATED runZipStream "Please use 'drain . fromZipSerial instead." #-} runZipStream :: Monad m => ZipSerialM m a -> m () -runZipStream = P.drain . K.adapt +runZipStream = P.drain . adapt -- | Same as @drain . zippingAsync@. -- -- @since 0.1.0 {-# DEPRECATED runZipAsync "Please use 'drain . fromZipAsync instead." #-} runZipAsync :: Monad m => ZipAsyncM m a -> m () -runZipAsync = P.drain . K.adapt +runZipAsync = P.drain . adapt {- -- | Same as "Streamly.Prelude.foldWith". @@ -451,7 +450,7 @@ forEachWith = P.forEachWith -- @since 0.2.0 {-# INLINABLE mkAsync #-} mkAsync :: (IsStream t, MonadAsync m) => t m a -> m (t m a) -mkAsync = return . Async.mkAsync +mkAsync = return . Transform.mkAsync ------------------------------------------------------------------------------ -- Documentation diff --git a/src/Streamly/Internal/Data/Array.hs b/src/Streamly/Internal/Data/Array.hs index 456bba904e..bca0393cb2 100644 --- a/src/Streamly/Internal/Data/Array.hs +++ b/src/Streamly/Internal/Data/Array.hs @@ -56,8 +56,7 @@ import GHC.Base (Int(..)) import GHC.IO (unsafePerformIO) import Streamly.Internal.Data.Fold.Type (Fold(..)) -import Streamly.Internal.Data.Stream.StreamK.Type (IsStream) -import Streamly.Internal.Data.Stream.Serial (SerialT) +import Streamly.Internal.Data.Stream.Serial (SerialT(..)) import Streamly.Internal.Data.Tuple.Strict (Tuple'(..), Tuple3'(..)) import Streamly.Internal.Data.Unfold.Type (Unfold(..)) @@ -131,13 +130,13 @@ fromStreamD = D.fold write {-# INLINE fromStreamN #-} fromStreamN :: MonadIO m => Int -> SerialT m a -> m (Array a) -fromStreamN n m = do +fromStreamN n (SerialT m) = do when (n < 0) $ error "fromStreamN: negative write count specified" - fromStreamDN n $ D.toStreamD m + fromStreamDN n $ D.fromStreamK m {-# INLINE fromStream #-} fromStream :: MonadIO m => SerialT m a -> m (Array a) -fromStream m = fromStreamD $ D.toStreamD m +fromStream (SerialT m) = fromStreamD $ D.fromStreamK m {-# INLINABLE fromListN #-} fromListN :: Int -> [a] -> Array a @@ -196,12 +195,12 @@ toStreamDRev arr = D.Stream step (length arr - 1) (# x #) -> D.Yield x (I# i - 1) {-# INLINE_EARLY toStream #-} -toStream :: (Monad m, IsStream t) => Array a -> t m a -toStream = D.fromStreamD . toStreamD +toStream :: Monad m => Array a -> SerialT m a +toStream = SerialT . D.toStreamK . toStreamD {-# INLINE_EARLY toStreamRev #-} -toStreamRev :: (Monad m, IsStream t) => Array a -> t m a -toStreamRev = D.fromStreamD . toStreamDRev +toStreamRev :: Monad m => Array a -> SerialT m a +toStreamRev = SerialT . D.toStreamK . toStreamDRev ------------------------------------------------------------------------------- -- Elimination - using Folds diff --git a/src/Streamly/Internal/Data/Array/Foreign.hs b/src/Streamly/Internal/Data/Array/Foreign.hs index 416ad58c8f..3151899f2c 100644 --- a/src/Streamly/Internal/Data/Array/Foreign.hs +++ b/src/Streamly/Internal/Data/Array/Foreign.hs @@ -141,7 +141,7 @@ import Streamly.Internal.BaseCompat import Streamly.Internal.Data.Array.Foreign.Type (Array(..), length) import Streamly.Internal.Data.Fold.Type (Fold(..)) import Streamly.Internal.Data.Producer.Type (Producer) -import Streamly.Internal.Data.Stream.Serial (SerialT) +import Streamly.Internal.Data.Stream.Serial (SerialT(..)) import Streamly.Internal.Data.Tuple.Strict (Tuple3'(..)) import Streamly.Internal.Data.Unfold.Type (Unfold(..)) import Streamly.Internal.System.IO (unsafeInlineIO) @@ -150,7 +150,6 @@ import qualified Streamly.Internal.Data.Array.Foreign.Mut.Type as MA import qualified Streamly.Internal.Data.Array.Foreign.Type as A import qualified Streamly.Internal.Data.Fold as FL import qualified Streamly.Internal.Data.Stream.Prelude as P -import qualified Streamly.Internal.Data.Stream.Serial as Serial import qualified Streamly.Internal.Data.Stream.StreamD as D import qualified Streamly.Internal.Data.Unfold as Unfold import qualified Streamly.Internal.Data.Producer.Type as Producer @@ -176,9 +175,9 @@ plusForeignPtr (ForeignPtr addr c) (I# d) = ForeignPtr (plusAddr# addr d) c -- /Pre-release/ {-# INLINE fromStreamN #-} fromStreamN :: (MonadIO m, Storable a) => Int -> SerialT m a -> m (Array a) -fromStreamN n m = do +fromStreamN n (SerialT m) = do when (n < 0) $ error "writeN: negative write count specified" - A.fromStreamDN n $ D.toStreamD m + A.fromStreamDN n $ D.fromStreamK m -- | Create an 'Array' from a stream. This is useful when we want to create a -- single array from a stream of unknown size. 'writeN' is at least twice @@ -191,8 +190,8 @@ fromStreamN n m = do -- /Pre-release/ {-# INLINE fromStream #-} fromStream :: (MonadIO m, Storable a) => SerialT m a -> m (Array a) -fromStream = P.fold A.write --- write m = A.fromStreamD $ D.toStreamD m +fromStream (SerialT m) = P.fold A.write m +-- write m = A.fromStreamD $ D.fromStreamK m ------------------------------------------------------------------------------- -- Elimination @@ -454,7 +453,7 @@ streamTransform :: forall m a b. (MonadIO m, Storable a, Storable b) => (SerialT m a -> SerialT m b) -> Array a -> m (Array b) streamTransform f arr = P.fold (A.toArrayMinChunk (alignment (undefined :: a)) (length arr)) - $ f (A.toStream arr) + $ getSerialT $ f (A.toStream arr) ------------------------------------------------------------------------------- -- Casts @@ -527,7 +526,7 @@ unsafeAsCString arr act = do -- /Pre-release/ {-# INLINE fold #-} fold :: forall m a b. (MonadIO m, Storable a) => Fold m a b -> Array a -> m b -fold f arr = P.fold f (A.toStream arr :: Serial.SerialT m a) +fold f arr = P.fold f (getSerialT (A.toStream arr)) -- | Fold an array using a stream fold operation. -- diff --git a/src/Streamly/Internal/Data/Array/Foreign/Mut/Type.hs b/src/Streamly/Internal/Data/Array/Foreign/Mut/Type.hs index 60409441fa..5cce7f5127 100644 --- a/src/Streamly/Internal/Data/Array/Foreign/Mut/Type.hs +++ b/src/Streamly/Internal/Data/Array/Foreign/Mut/Type.hs @@ -686,7 +686,7 @@ toStreamD Array{..} = return $ D.Yield x (p `plusPtr` sizeOf (undefined :: a)) {-# INLINE toStreamK #-} -toStreamK :: forall t m a. (K.IsStream t, Storable a) => Array a -> t m a +toStreamK :: forall m a. Storable a => Array a -> K.Stream m a toStreamK Array{..} = let p = unsafeForeignPtrToPtr aStart in go p @@ -726,7 +726,7 @@ toStreamDRev Array{..} = return $ D.Yield x (p `plusPtr` negate (sizeOf (undefined :: a))) {-# INLINE toStreamKRev #-} -toStreamKRev :: forall t m a. (K.IsStream t, Storable a) => Array a -> t m a +toStreamKRev :: forall m a. Storable a => Array a -> K.Stream m a toStreamKRev Array {..} = let p = aEnd `plusPtr` negate (sizeOf (undefined :: a)) in go p diff --git a/src/Streamly/Internal/Data/Array/Foreign/Type.hs b/src/Streamly/Internal/Data/Array/Foreign/Type.hs index 5072270129..c9de74dc31 100644 --- a/src/Streamly/Internal/Data/Array/Foreign/Type.hs +++ b/src/Streamly/Internal/Data/Array/Foreign/Type.hs @@ -91,6 +91,7 @@ import GHC.ForeignPtr (touchForeignPtr, unsafeForeignPtrToPtr) import GHC.IO (unsafePerformIO) import GHC.Ptr (Ptr(..)) import Streamly.Internal.Data.Fold.Type (Fold(..)) +import Streamly.Internal.Data.Stream.Serial (SerialT(..)) import Streamly.Internal.Data.Unfold.Type (Unfold(..)) import Text.Read (readPrec, readListPrec, readListPrecDefault) @@ -429,7 +430,7 @@ toStreamD :: forall m a. (Monad m, Storable a) => Array a -> D.Stream m a toStreamD arr = MA.toStreamD (unsafeThaw arr) {-# INLINE toStreamK #-} -toStreamK :: forall t m a. (K.IsStream t, Storable a) => Array a -> t m a +toStreamK :: forall m a. Storable a => Array a -> K.Stream m a toStreamK arr = MA.toStreamK (unsafeThaw arr) {-# INLINE_NORMAL toStreamDRev #-} @@ -437,15 +438,15 @@ toStreamDRev :: forall m a. (Monad m, Storable a) => Array a -> D.Stream m a toStreamDRev arr = MA.toStreamDRev (unsafeThaw arr) {-# INLINE toStreamKRev #-} -toStreamKRev :: forall t m a. (K.IsStream t, Storable a) => Array a -> t m a +toStreamKRev :: forall m a. Storable a => Array a -> K.Stream m a toStreamKRev arr = MA.toStreamKRev (unsafeThaw arr) -- | Convert an 'Array' into a stream. -- -- /Pre-release/ {-# INLINE_EARLY toStream #-} -toStream :: (Monad m, K.IsStream t, Storable a) => Array a -> t m a -toStream = D.fromStreamD . toStreamD +toStream :: (Monad m, Storable a) => Array a -> SerialT m a +toStream = SerialT . D.toStreamK . toStreamD -- XXX add fallback to StreamK rule -- {-# RULES "Streamly.Array.read fallback to StreamK" [1] -- forall a. S.readK (read a) = K.fromArray a #-} @@ -454,8 +455,8 @@ toStream = D.fromStreamD . toStreamD -- -- /Pre-release/ {-# INLINE_EARLY toStreamRev #-} -toStreamRev :: (Monad m, K.IsStream t, Storable a) => Array a -> t m a -toStreamRev = D.fromStreamD . toStreamDRev +toStreamRev :: (Monad m, Storable a) => Array a -> SerialT m a +toStreamRev = SerialT . D.toStreamK . toStreamDRev -- XXX add fallback to StreamK rule -- {-# RULES "Streamly.Array.readRev fallback to StreamK" [1] diff --git a/src/Streamly/Internal/Data/Array/Prim/TypesInclude.hs b/src/Streamly/Internal/Data/Array/Prim/TypesInclude.hs index e82f750647..d80b6c2262 100644 --- a/src/Streamly/Internal/Data/Array/Prim/TypesInclude.hs +++ b/src/Streamly/Internal/Data/Array/Prim/TypesInclude.hs @@ -25,7 +25,7 @@ import qualified GHC.Exts as Exts import qualified Prelude as P import qualified Streamly.Internal.Data.Fold as FL import qualified Streamly.Internal.Data.Stream.StreamD.Type as D -import qualified Streamly.Internal.Data.Stream.StreamK as K +import qualified Streamly.Internal.Data.Stream.StreamK.Type as K import GHC.Exts hiding (fromListN, fromList, toList) import Prelude hiding (length, unlines, foldr) @@ -398,7 +398,7 @@ toStreamD arr = D.Stream step 0 step _ i = return $ D.Yield (unsafeIndex arr i) (i + 1) {-# INLINE toStreamK #-} -toStreamK :: (K.IsStream t, Prim a) => Array a -> t m a +toStreamK :: Prim a => Array a -> K.Stream m a toStreamK arr = go 0 where @@ -423,7 +423,7 @@ toStreamDRev arr = D.Stream step (length arr - 1) step _ i = return $ D.Yield (unsafeIndex arr i) (i - 1) {-# INLINE toStreamKRev #-} -toStreamKRev :: (K.IsStream t, Prim a) => Array a -> t m a +toStreamKRev :: Prim a => Array a -> K.Stream m a toStreamKRev arr = go (length arr - 1) where diff --git a/src/Streamly/Internal/Data/Array/PrimInclude.hs b/src/Streamly/Internal/Data/Array/PrimInclude.hs index f3d77c5212..9f3d7c3a83 100644 --- a/src/Streamly/Internal/Data/Array/PrimInclude.hs +++ b/src/Streamly/Internal/Data/Array/PrimInclude.hs @@ -9,13 +9,10 @@ import Control.Monad.IO.Class (MonadIO(..)) import Data.Primitive.Types (Prim(..)) import Streamly.Internal.Data.Fold.Type (Fold(..)) import Streamly.Internal.Data.Unfold.Type (Unfold(..)) -import Streamly.Internal.Data.Stream.Serial (SerialT) -import Streamly.Internal.Data.Stream.StreamK.Type (IsStream) +import Streamly.Internal.Data.Stream.Serial (SerialT(..)) import qualified Streamly.Internal.Data.Stream.Prelude as P -import qualified Streamly.Internal.Data.Stream.Serial as Serial import qualified Streamly.Internal.Data.Stream.StreamD as D -import qualified Streamly.Internal.Data.Stream.StreamK as K import Prelude hiding (length, null, last, map, (!!), read, concat) @@ -30,9 +27,9 @@ import Prelude hiding (length, null, last, map, (!!), read, concat) -- /Pre-release/ {-# INLINE fromStreamN #-} fromStreamN :: (MonadIO m, Prim a) => Int -> SerialT m a -> m (Array a) -fromStreamN n m = do +fromStreamN n (SerialT m) = do when (n < 0) $ error "writeN: negative write count specified" - A.fromStreamDN n $ D.toStreamD m + A.fromStreamDN n $ D.fromStreamK m -- | Create an 'Array' from a stream. This is useful when we want to create a -- single array from a stream of unknown size. 'writeN' is at least twice @@ -45,7 +42,7 @@ fromStreamN n m = do -- /Pre-release/ {-# INLINE fromStream #-} fromStream :: (MonadIO m, Prim a) => SerialT m a -> m (Array a) -fromStream = P.fold A.write +fromStream (SerialT m) = P.fold A.write m -- write m = A.fromStreamD $ D.toStreamD m ------------------------------------------------------------------------------- @@ -56,8 +53,8 @@ fromStream = P.fold A.write -- -- /Pre-release/ {-# INLINE_EARLY toStream #-} -toStream :: (MonadIO m, K.IsStream t, Prim a) => Array a -> t m a -toStream = D.fromStreamD . A.toStreamD +toStream :: (MonadIO m, Prim a) => Array a -> SerialT m a +toStream = SerialT . D.toStreamK . A.toStreamD -- XXX add fallback to StreamK rule -- {-# RULES "Streamly.Array.read fallback to StreamK" [1] -- forall a. S.readK (read a) = K.fromArray a #-} @@ -66,8 +63,8 @@ toStream = D.fromStreamD . A.toStreamD -- -- /Pre-release/ {-# INLINE_EARLY toStreamRev #-} -toStreamRev :: (MonadIO m, IsStream t, Prim a) => Array a -> t m a -toStreamRev = D.fromStreamD . A.toStreamDRev +toStreamRev :: (MonadIO m, Prim a) => Array a -> SerialT m a +toStreamRev = SerialT . D.toStreamK . A.toStreamDRev -- XXX add fallback to StreamK rule -- {-# RULES "Streamly.Array.readRev fallback to StreamK" [1] -- forall a. S.toStreamK (readRev a) = K.revFromArray a #-} @@ -128,7 +125,7 @@ null arr = length arr == 0 -- /Pre-release/ {-# INLINE fold #-} fold :: forall m a b. (MonadIO m, Prim a) => Fold m a b -> Array a -> m b -fold f arr = P.fold f (toStream arr :: Serial.SerialT m a) +fold f arr = P.fold f (getSerialT (toStream arr)) -- | Fold an array using a stream fold operation. -- @@ -170,10 +167,11 @@ last arr = readIndex arr (length arr - 1) -- -- /Pre-release/ {-# INLINE concat #-} -concat :: (IsStream t, MonadIO m, Prim a) => t m (Array a) -> t m a +concat :: (MonadIO m, Prim a) => SerialT m (Array a) -> SerialT m a -- concat m = D.fromStreamD $ A.flattenArrays (D.toStreamD m) -- concat m = D.fromStreamD $ D.concatMap A.toStreamD (D.toStreamD m) -concat m = D.fromStreamD $ D.unfoldMany read (D.toStreamD m) +concat (SerialT m) = + SerialT $ D.toStreamK $ D.unfoldMany read (D.fromStreamK m) -- | Coalesce adjacent arrays in incoming stream to form bigger arrays of a -- maximum specified size in bytes. @@ -182,4 +180,5 @@ concat m = D.fromStreamD $ D.unfoldMany read (D.toStreamD m) {-# INLINE compact #-} compact :: (MonadIO m, Prim a) => Int -> SerialT m (Array a) -> SerialT m (Array a) -compact n xs = D.fromStreamD $ A.packArraysChunksOf n (D.toStreamD xs) +compact n (SerialT xs) = + SerialT $ D.toStreamK $ A.packArraysChunksOf n (D.fromStreamK xs) diff --git a/src/Streamly/Internal/Data/Array/Stream/Foreign.hs b/src/Streamly/Internal/Data/Array/Stream/Foreign.hs index ad544cea08..2d262b1c8a 100644 --- a/src/Streamly/Internal/Data/Array/Stream/Foreign.hs +++ b/src/Streamly/Internal/Data/Array/Stream/Foreign.hs @@ -72,7 +72,8 @@ import Streamly.Internal.Data.Array.Foreign.Type (Array(..)) import Streamly.Internal.Data.Fold.Type (Fold(..)) import Streamly.Internal.Data.Parser (ParseError(..)) import Streamly.Internal.Data.Stream.Serial (SerialT) -import Streamly.Internal.Data.Stream.StreamK.Type (IsStream) +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream, fromStreamD, toStreamD) import Streamly.Internal.Data.SVar (adaptState, defState) import Streamly.Internal.Data.Array.Foreign.Mut.Type (memcpy, bytesToElemCount) import Streamly.Internal.System.IO (mkChunkSizeKB) @@ -102,7 +103,7 @@ import qualified Streamly.Internal.Data.Stream.StreamD as D {-# INLINE arraysOf #-} arraysOf :: (IsStream t, MonadIO m, Storable a) => Int -> t m a -> t m (Array a) -arraysOf n str = D.fromStreamD $ A.arraysOf n (D.toStreamD str) +arraysOf n str = fromStreamD $ A.arraysOf n (toStreamD str) ------------------------------------------------------------------------------- -- Append @@ -121,9 +122,9 @@ arraysOf n str = D.fromStreamD $ A.arraysOf n (D.toStreamD str) -- @since 0.7.0 {-# INLINE concat #-} concat :: (IsStream t, MonadIO m, Storable a) => t m (Array a) -> t m a --- concat m = D.fromStreamD $ A.flattenArrays (D.toStreamD m) --- concat m = D.fromStreamD $ D.concatMap A.toStreamD (D.toStreamD m) -concat m = D.fromStreamD $ D.unfoldMany A.read (D.toStreamD m) +-- concat m = fromStreamD $ A.flattenArrays (toStreamD m) +-- concat m = fromStreamD $ D.concatMap A.toStreamD (toStreamD m) +concat m = fromStreamD $ D.unfoldMany A.read (toStreamD m) -- | Convert a stream of arrays into a stream of their elements reversing the -- contents of each array before flattening. @@ -133,7 +134,7 @@ concat m = D.fromStreamD $ D.unfoldMany A.read (D.toStreamD m) -- @since 0.7.0 {-# INLINE concatRev #-} concatRev :: (IsStream t, MonadIO m, Storable a) => t m (Array a) -> t m a -concatRev m = D.fromStreamD $ A.flattenArraysRev (D.toStreamD m) +concatRev m = fromStreamD $ A.flattenArraysRev (toStreamD m) ------------------------------------------------------------------------------- -- Intersperse and append @@ -159,7 +160,7 @@ intercalateSuffix = S.intercalateSuffix A.read {-# INLINE interposeSuffix #-} interposeSuffix :: (MonadIO m, IsStream t, Storable a) => a -> t m (Array a) -> t m a --- interposeSuffix x = D.fromStreamD . A.unlines x . D.toStreamD +-- interposeSuffix x = fromStreamD . A.unlines x . toStreamD interposeSuffix x = S.interposeSuffix x A.read data FlattenState s a = @@ -220,7 +221,7 @@ lpackArraysChunksOf n fld = {-# INLINE compact #-} compact :: (MonadIO m, Storable a) => Int -> SerialT m (Array a) -> SerialT m (Array a) -compact n xs = D.fromStreamD $ packArraysChunksOf n (D.toStreamD xs) +compact n xs = fromStreamD $ packArraysChunksOf n (toStreamD xs) ------------------------------------------------------------------------------- -- Split @@ -294,7 +295,7 @@ splitOn -> t m (Array Word8) -> t m (Array Word8) splitOn byte s = - D.fromStreamD $ D.splitInnerBy (A.breakOn byte) A.spliceTwo $ D.toStreamD s + fromStreamD $ D.splitInnerBy (A.breakOn byte) A.spliceTwo $ toStreamD s {-# INLINE splitOnSuffix #-} splitOnSuffix @@ -302,9 +303,9 @@ splitOnSuffix => Word8 -> t m (Array Word8) -> t m (Array Word8) --- splitOn byte s = D.fromStreamD $ A.splitOn byte $ D.toStreamD s +-- splitOn byte s = fromStreamD $ A.splitOn byte $ toStreamD s splitOnSuffix byte s = - D.fromStreamD $ D.splitInnerBySuffix (A.breakOn byte) A.spliceTwo $ D.toStreamD s + fromStreamD $ D.splitInnerBySuffix (A.breakOn byte) A.spliceTwo $ toStreamD s ------------------------------------------------------------------------------- -- Elimination - Running folds @@ -362,7 +363,7 @@ fold :: => FL.Fold m a b -> SerialT m (A.Array a) -> m (b, SerialT m (A.Array a)) -fold f s = fmap D.fromStreamD <$> foldD f (D.toStreamD s) +fold f s = fmap fromStreamD <$> foldD f (toStreamD s) ------------------------------------------------------------------------------- -- Fold to a single Array @@ -613,7 +614,7 @@ parse :: => PRD.Parser m a b -> SerialT m (A.Array a) -> m (b, SerialT m (A.Array a)) -parse p s = fmap D.fromStreamD <$> parseD p (D.toStreamD s) +parse p s = fmap fromStreamD <$> parseD p (toStreamD s) ------------------------------------------------------------------------------- -- Elimination - Running Array Folds and parsers @@ -695,7 +696,7 @@ parseArr :: => ASF.Parser m a b -> SerialT m (A.Array a) -> m (b, SerialT m (A.Array a)) -parseArr p s = fmap D.fromStreamD <$> parseD p (D.toStreamD s) +parseArr p s = fmap fromStreamD <$> parseD p (toStreamD s) -} -- | Fold an array stream using the supplied array stream 'Fold'. @@ -705,7 +706,7 @@ parseArr p s = fmap D.fromStreamD <$> parseD p (D.toStreamD s) {-# INLINE foldArr #-} foldArr :: (MonadIO m, MonadThrow m, Storable a) => ASF.Fold m a b -> SerialT m (A.Array a) -> m b -foldArr (ASF.Fold p) s = fst <$> parseArrD p (D.toStreamD s) +foldArr (ASF.Fold p) s = fst <$> parseArrD p (toStreamD s) -- | Like 'fold' but also returns the remaining stream. -- @@ -714,7 +715,7 @@ foldArr (ASF.Fold p) s = fst <$> parseArrD p (D.toStreamD s) {-# INLINE foldArr_ #-} foldArr_ :: (MonadIO m, MonadThrow m, Storable a) => ASF.Fold m a b -> SerialT m (A.Array a) -> m (b, SerialT m (A.Array a)) -foldArr_ (ASF.Fold p) s = second D.fromStreamD <$> parseArrD p (D.toStreamD s) +foldArr_ (ASF.Fold p) s = second fromStreamD <$> parseArrD p (toStreamD s) {-# ANN type ParseChunksState Fuse #-} data ParseChunksState x inpBuf st pst = @@ -856,4 +857,4 @@ foldArrMany => ASF.Fold m a b -> t m (Array a) -> t m b -foldArrMany p m = D.fromStreamD $ foldArrManyD p (D.toStreamD m) +foldArrMany p m = fromStreamD $ foldArrManyD p (toStreamD m) diff --git a/src/Streamly/Internal/Data/Array/Stream/Mut/Foreign.hs b/src/Streamly/Internal/Data/Array/Stream/Mut/Foreign.hs index f55f92ae38..60aa764d17 100644 --- a/src/Streamly/Internal/Data/Array/Stream/Mut/Foreign.hs +++ b/src/Streamly/Internal/Data/Array/Stream/Mut/Foreign.hs @@ -32,8 +32,9 @@ import Data.Bifunctor (first) import Foreign.Storable (Storable(..)) import Streamly.Internal.Data.Array.Foreign.Mut.Type (Array(..)) import Streamly.Internal.Data.Fold.Type (Fold(..)) -import Streamly.Internal.Data.Stream.Serial (SerialT) -import Streamly.Internal.Data.Stream.StreamK.Type (IsStream) +import Streamly.Internal.Data.Stream.Serial (SerialT(..)) +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream, fromStreamD, toStreamD) import Streamly.Internal.Data.Tuple.Strict (Tuple'(..)) import qualified Streamly.Internal.Data.Array.Foreign.Mut.Type as MArray @@ -51,7 +52,7 @@ import qualified Streamly.Internal.Data.Stream.StreamD as D {-# INLINE arraysOf #-} arraysOf :: (IsStream t, MonadIO m, Storable a) => Int -> t m a -> t m (Array a) -arraysOf n = D.fromStreamD . MArray.arraysOf n . D.toStreamD +arraysOf n = fromStreamD . MArray.arraysOf n . toStreamD ------------------------------------------------------------------------------- -- Compact @@ -190,7 +191,8 @@ lpackArraysChunksOf n (Fold step1 initial1 extract1) = {-# INLINE compact #-} compact :: (MonadIO m, Storable a) => Int -> SerialT m (Array a) -> SerialT m (Array a) -compact n xs = D.fromStreamD $ packArraysChunksOf n (D.toStreamD xs) +compact n (SerialT xs) = + SerialT $ D.toStreamK $ packArraysChunksOf n (D.fromStreamK xs) -- See lpackArraysChunksOf/packArraysChunksOf to implement this. -- @@ -212,7 +214,8 @@ compactLEFold = undefined -- /Internal/ compactLE :: (MonadIO m {-, Storable a-}) => Int -> SerialT m (Array a) -> SerialT m (Array a) -compactLE n xs = D.fromStreamD $ D.foldMany (compactLEFold n) (D.toStreamD xs) +compactLE n (SerialT xs) = + SerialT $ D.toStreamK $ D.foldMany (compactLEFold n) (D.fromStreamK xs) -- | Like 'compactLE' but generates arrays of exactly equal to the size -- specified except for the last array in the stream which could be shorter. @@ -222,7 +225,7 @@ compactLE n xs = D.fromStreamD $ D.foldMany (compactLEFold n) (D.toStreamD xs) compactEQ :: -- (MonadIO m, Storable a) => Int -> SerialT m (Array a) -> SerialT m (Array a) compactEQ _n _xs = undefined - -- D.fromStreamD $ D.foldMany (compactEQFold n) (D.toStreamD xs) + -- IsStream.fromStreamD $ D.foldMany (compactEQFold n) (IsStream.toStreamD xs) -- | Like 'compactLE' but generates arrays of size greater than or equal to the -- specified except for the last array in the stream which could be shorter. @@ -232,4 +235,4 @@ compactEQ _n _xs = undefined compactGE :: -- (MonadIO m, Storable a) => Int -> SerialT m (Array a) -> SerialT m (Array a) compactGE _n _xs = undefined - -- D.fromStreamD $ D.foldMany (compactGEFold n) (D.toStreamD xs) + -- IsStream.fromStreamD $ D.foldMany (compactGEFold n) (IsStream.toStreamD xs) diff --git a/src/Streamly/Internal/Data/Fold.hs b/src/Streamly/Internal/Data/Fold.hs index dddaf4353f..e7b997e402 100644 --- a/src/Streamly/Internal/Data/Fold.hs +++ b/src/Streamly/Internal/Data/Fold.hs @@ -143,7 +143,6 @@ module Streamly.Internal.Data.Fold -- ** Trimming , take - , takeInterval -- By elements , takeEndBy @@ -215,7 +214,6 @@ module Streamly.Internal.Data.Fold -- ** Splitting , many - , intervalsOf , chunksOf , chunksBetween @@ -252,12 +250,12 @@ import Streamly.Internal.Data.Either.Strict (Either'(..), fromLeft', fromRight', isLeft', isRight') import Streamly.Internal.Data.Pipe.Type (Pipe (..), PipeState(..)) import Streamly.Internal.Data.Tuple.Strict (Tuple'(..), Tuple3'(..)) -import Streamly.Internal.Data.Stream.Serial (SerialT) +import Streamly.Internal.Data.Stream.Serial (SerialT(..)) import qualified Data.Map.Strict as Map import qualified Streamly.Internal.Data.Pipe.Type as Pipe -- import qualified Streamly.Internal.Data.Stream.IsStream.Enumeration as Stream -import qualified Streamly.Internal.Data.Stream.StreamK as K +import qualified Streamly.Internal.Data.Stream.StreamK.Type as K import qualified Prelude import Prelude hiding @@ -1690,7 +1688,7 @@ chunksBetween _low _high _f1 _f2 = undefined -- /Pre-release/ {-# INLINE toStream #-} toStream :: Monad m => Fold m a (SerialT Identity a) -toStream = foldr K.cons K.nil +toStream = fmap SerialT $ foldr K.cons K.nil -- This is more efficient than 'toStream'. toStream is exactly the same as -- reversing the stream after toStreamRev. @@ -1708,4 +1706,4 @@ toStream = foldr K.cons K.nil -- xn : ... : x2 : x1 : [] {-# INLINABLE toStreamRev #-} toStreamRev :: Monad m => Fold m a (SerialT Identity a) -toStreamRev = foldl' (flip K.cons) K.nil +toStreamRev = fmap SerialT $ foldl' (flip K.cons) K.nil diff --git a/src/Streamly/Internal/Data/Fold/Async.hs b/src/Streamly/Internal/Data/Fold/Async.hs new file mode 100644 index 0000000000..175f4f48af --- /dev/null +++ b/src/Streamly/Internal/Data/Fold/Async.hs @@ -0,0 +1,120 @@ +-- | +-- Module : Streamly.Internal.Data.Fold.Async +-- Copyright : (c) 2019 Composewell Technologies +-- License : BSD-3-Clause +-- Maintainer : streamly@composewell.com +-- Stability : experimental +-- Portability : GHC +-- +module Streamly.Internal.Data.Fold.Async + ( + -- * Trimming + takeInterval + + -- * Splitting + , intervalsOf + ) +where + +import Control.Concurrent (threadDelay, forkIO, killThread) +import Control.Concurrent.MVar (MVar, newMVar, swapMVar, readMVar) +import Control.Exception (SomeException(..), catch, mask) +import Control.Monad (void) +import Control.Monad.IO.Class (MonadIO(..)) +import Control.Monad.Trans.Control (control) +import Streamly.Internal.Control.Concurrent (MonadAsync) +import Streamly.Internal.Data.Tuple.Strict (Tuple3'(..)) + +import Streamly.Internal.Data.Fold.Type + +-- $setup +-- >>> :m +-- >>> import qualified Streamly.Prelude as Stream +-- >>> import qualified Streamly.Data.Fold as Fold +-- >>> import qualified Streamly.Internal.Data.Fold.Async as Fold + +-- XXX We can use asyncClock here. A parser can be used to return an input that +-- arrives after the timeout. +-- XXX If n is 0 return immediately in initial. +-- XXX we should probably discard the input received after the timeout like +-- takeEndBy_. +-- +-- | @takeInterval n fold@ uses @fold@ to fold the input items arriving within +-- a window of first @n@ seconds. +-- +-- >>> input = Stream.delay 0.1 $ Stream.fromList [1..] +-- >>> Stream.fold (Fold.takeInterval 1.0 Fold.toList) input +-- [1,2,3,4,5,6,7,8,9,10,11] +-- +-- Stops when @fold@ stops or when the timeout occurs. Note that the fold needs +-- an input after the timeout to stop. For example, if no input is pushed to +-- the fold until one hour after the timeout had occurred, then the fold will +-- be done only after consuming that input. +-- +-- /Pre-release/ +-- +{-# INLINE takeInterval #-} +takeInterval :: MonadAsync m => Double -> Fold m a b -> Fold m a b +takeInterval n (Fold step initial done) = Fold step' initial' done' + + where + + initial' = do + res <- initial + case res of + Partial s -> do + mv <- liftIO $ newMVar False + t <- + control $ \run -> + mask $ \restore -> do + tid <- + forkIO + $ catch + (restore $ void $ run (timerThread mv)) + (handleChildException mv) + run (return tid) + return $ Partial $ Tuple3' s mv t + Done b -> return $ Done b + + step' (Tuple3' s mv t) a = do + val <- liftIO $ readMVar mv + if val + then do + res <- step s a + case res of + Partial sres -> Done <$> done sres + Done bres -> return $ Done bres + else do + res <- step s a + case res of + Partial fs -> return $ Partial $ Tuple3' fs mv t + Done b -> liftIO (killThread t) >> return (Done b) + + done' (Tuple3' s _ _) = done s + + timerThread mv = do + liftIO $ threadDelay (round $ n * 1000000) + -- Use IORef + CAS? instead of MVar since its a Bool? + liftIO $ void $ swapMVar mv True + + handleChildException :: MVar Bool -> SomeException -> IO () + handleChildException mv _ = void $ swapMVar mv True + +-- For example, we can copy and distribute a stream to multiple folds where +-- each fold can group the input differently e.g. by one second, one minute and +-- one hour windows respectively and fold each resulting stream of folds. + +-- | Group the input stream into windows of n second each using the first fold +-- and then fold the resulting groups using the second fold. +-- +-- >>> intervals = Fold.intervalsOf 0.5 Fold.toList Fold.toList +-- >>> Stream.fold intervals $ Stream.delay 0.2 $ Stream.fromList [1..10] +-- [[1,2,3,4],[5,6,7],[8,9,10]] +-- +-- > intervalsOf n split = many (takeInterval n split) +-- +-- /Pre-release/ +-- +{-# INLINE intervalsOf #-} +intervalsOf :: MonadAsync m => Double -> Fold m a b -> Fold m b c -> Fold m a c +intervalsOf n split = many (takeInterval n split) diff --git a/src/Streamly/Internal/Data/Fold/Type.hs b/src/Streamly/Internal/Data/Fold/Type.hs index 09ede87653..95ea7111a6 100644 --- a/src/Streamly/Internal/Data/Fold/Type.hs +++ b/src/Streamly/Internal/Data/Fold/Type.hs @@ -228,7 +228,6 @@ module Streamly.Internal.Data.Fold.Type -- ** Trimming , take - , takeInterval -- ** Serial Append , serialWith @@ -248,7 +247,6 @@ module Streamly.Internal.Data.Fold.Type , many , manyPost , chunksOf - , intervalsOf -- ** Nesting , concatMap @@ -265,18 +263,12 @@ module Streamly.Internal.Data.Fold.Type ) where -import Control.Monad (void, (>=>)) -import Control.Concurrent (threadDelay, forkIO, killThread) -import Control.Concurrent.MVar (MVar, newMVar, swapMVar, readMVar) -import Control.Exception (SomeException(..), catch, mask) -import Control.Monad.IO.Class (MonadIO(..)) -import Control.Monad.Trans.Control (control) +import Control.Monad ((>=>)) import Data.Bifunctor (Bifunctor(..)) import Data.Maybe (isJust, fromJust) import Fusion.Plugin.Types (Fuse(..)) import Streamly.Internal.Data.Maybe.Strict (Maybe'(..), toMaybe) import Streamly.Internal.Data.Tuple.Strict (Tuple'(..), Tuple3'(..)) -import Streamly.Internal.Control.Concurrent (MonadAsync) import Prelude hiding (concatMap, filter, foldr, map, take) @@ -1358,88 +1350,3 @@ chunksOf2 n (Fold step1 initial1 extract1) (Fold2 step2 inject2 extract2) = else extract1 r1 >>= step2 r2 >>= loopUntilPartial extract' (Tuple3' _ r1 r2) = extract1 r1 >>= step2 r2 >>= extract2 - --- XXX We can use asyncClock here. A parser can be used to return an input that --- arrives after the timeout. --- XXX If n is 0 return immediately in initial. --- XXX we should probably discard the input received after the timeout like --- takeEndBy_. --- --- | @takeInterval n fold@ uses @fold@ to fold the input items arriving within --- a window of first @n@ seconds. --- --- >>> Stream.fold (Fold.takeInterval 1.0 Fold.toList) $ Stream.delay 0.1 $ Stream.fromList [1..] --- [1,2,3,4,5,6,7,8,9,10,11] --- --- Stops when @fold@ stops or when the timeout occurs. Note that the fold needs --- an input after the timeout to stop. For example, if no input is pushed to --- the fold until one hour after the timeout had occurred, then the fold will --- be done only after consuming that input. --- --- /Pre-release/ --- -{-# INLINE takeInterval #-} -takeInterval :: MonadAsync m => Double -> Fold m a b -> Fold m a b -takeInterval n (Fold step initial done) = Fold step' initial' done' - - where - - initial' = do - res <- initial - case res of - Partial s -> do - mv <- liftIO $ newMVar False - t <- - control $ \run -> - mask $ \restore -> do - tid <- - forkIO - $ catch - (restore $ void $ run (timerThread mv)) - (handleChildException mv) - run (return tid) - return $ Partial $ Tuple3' s mv t - Done b -> return $ Done b - - step' (Tuple3' s mv t) a = do - val <- liftIO $ readMVar mv - if val - then do - res <- step s a - case res of - Partial sres -> Done <$> done sres - Done bres -> return $ Done bres - else do - res <- step s a - case res of - Partial fs -> return $ Partial $ Tuple3' fs mv t - Done b -> liftIO (killThread t) >> return (Done b) - - done' (Tuple3' s _ _) = done s - - timerThread mv = do - liftIO $ threadDelay (round $ n * 1000000) - -- Use IORef + CAS? instead of MVar since its a Bool? - liftIO $ void $ swapMVar mv True - - handleChildException :: MVar Bool -> SomeException -> IO () - handleChildException mv _ = void $ swapMVar mv True - --- For example, we can copy and distribute a stream to multiple folds where --- each fold can group the input differently e.g. by one second, one minute and --- one hour windows respectively and fold each resulting stream of folds. - --- | Group the input stream into windows of n second each using the first fold --- and then fold the resulting groups using the second fold. --- --- >>> intervals = Fold.intervalsOf 0.5 Fold.toList Fold.toList --- >>> Stream.fold intervals $ Stream.delay 0.2 $ Stream.fromList [1..10] --- [[1,2,3,4],[5,6,7],[8,9,10]] --- --- > intervalsOf n split = many (takeInterval n split) --- --- /Pre-release/ --- -{-# INLINE intervalsOf #-} -intervalsOf :: MonadAsync m => Double -> Fold m a b -> Fold m b c -> Fold m a c -intervalsOf n split = many (takeInterval n split) diff --git a/src/Streamly/Internal/Data/IORef/Prim.hs b/src/Streamly/Internal/Data/IORef/Prim.hs index e934c32f52..a088e5a7a9 100644 --- a/src/Streamly/Internal/Data/IORef/Prim.hs +++ b/src/Streamly/Internal/Data/IORef/Prim.hs @@ -35,7 +35,6 @@ module Streamly.Internal.Data.IORef.Prim -- * Read , readIORef - , toStream , toStreamD ) where @@ -48,7 +47,6 @@ import Data.Primitive.Types (Prim, sizeOf#, readByteArray#, writeByteArray#) import GHC.Exts (MutableByteArray#, newByteArray#, RealWorld) import GHC.IO (IO(..)) -import qualified Streamly.Internal.Data.Stream.StreamK.Type as K import qualified Streamly.Internal.Data.Stream.StreamD.Type as D -- | An 'IORef' holds a single 'Prim' value. @@ -100,11 +98,3 @@ toStreamD var = D.Stream step () {-# INLINE_LATE step #-} step _ () = liftIO (readIORef var) >>= \x -> return $ D.Yield x () - --- | Construct a stream by reading a 'Prim' 'IORef' repeatedly. --- --- /Pre-release/ --- -{-# INLINE toStream #-} -toStream :: (K.IsStream t, MonadIO m, Prim a) => IORef a -> t m a -toStream = D.fromStreamD . toStreamD diff --git a/src/Streamly/Internal/Data/List.hs b/src/Streamly/Internal/Data/List.hs index 390559d8ad..323fce98f6 100644 --- a/src/Streamly/Internal/Data/List.hs +++ b/src/Streamly/Internal/Data/List.hs @@ -72,11 +72,11 @@ import Data.Semigroup (Semigroup(..)) #endif import GHC.Exts (IsList(..), IsString(..)) -import Streamly.Internal.Data.Stream.Serial (SerialT) -import Streamly.Internal.Data.Stream.Zip (ZipSerialM) +import Streamly.Internal.Data.Stream.Serial (SerialT(..)) +import Streamly.Internal.Data.Stream.Zip (ZipSerialM(..)) -import qualified Streamly.Internal.Data.Stream.IsStream as Stream -import qualified Streamly.Internal.Data.Stream.StreamK as K +import qualified Streamly.Internal.Data.Stream.Serial as Serial +import qualified Streamly.Internal.Data.Stream.StreamK.Type as K -- We implement list as a newtype instead of a type synonym to make type -- inference easier when using -XOverloadedLists and -XOverloadedStrings. When @@ -103,15 +103,15 @@ newtype List a = List { toSerial :: SerialT Identity a } instance (a ~ Char) => IsString (List a) where {-# INLINE fromString #-} - fromString = List . Stream.fromList + fromString = List . fromList -- GHC versions 8.0 and below cannot derive IsList instance IsList (List a) where type (Item (List a)) = a {-# INLINE fromList #-} - fromList = List . Stream.fromList + fromList = List . fromList {-# INLINE toList #-} - toList = runIdentity . Stream.toList . toSerial + toList = toList . toSerial ------------------------------------------------------------------------------ -- Patterns @@ -128,8 +128,8 @@ instance IsList (List a) where -- -- @since 0.6.0 pattern Nil :: List a -pattern Nil <- (runIdentity . K.null . toSerial -> True) where - Nil = List K.nil +pattern Nil <- (runIdentity . K.null . getSerialT . toSerial -> True) where + Nil = List $ SerialT K.nil infixr 5 `Cons` @@ -139,9 +139,14 @@ infixr 5 `Cons` -- @since 0.6.0 pattern Cons :: a -> List a -> List a pattern Cons x xs <- - (fmap (second List) . runIdentity . K.uncons . toSerial - -> Just (x, xs)) where - Cons x xs = List $ K.cons x (toSerial xs) + (fmap (second (List . SerialT)) + . runIdentity . K.uncons . getSerialT . toSerial + -> Just (x, xs) + ) + + where + + Cons x xs = List $ Serial.cons x (toSerial xs) #if __GLASGOW_HASKELL__ >= 802 {-# COMPLETE Nil, Cons #-} @@ -165,24 +170,24 @@ newtype ZipList a = ZipList { toZipSerial :: ZipSerialM Identity a } instance (a ~ Char) => IsString (ZipList a) where {-# INLINE fromString #-} - fromString = ZipList . Stream.fromList + fromString = ZipList . fromList -- GHC versions 8.0 and below cannot derive IsList instance IsList (ZipList a) where type (Item (ZipList a)) = a {-# INLINE fromList #-} - fromList = ZipList . Stream.fromList + fromList = ZipList . fromList {-# INLINE toList #-} - toList = runIdentity . Stream.toList . K.adapt . toZipSerial + toList = toList . toZipSerial -- | Convert a 'ZipList' to a regular 'List' -- -- @since 0.6.0 fromZipList :: ZipList a -> List a -fromZipList = List . K.adapt . toZipSerial +fromZipList (ZipList zs) = List $ SerialT $ getZipSerialM zs -- | Convert a regular 'List' to a 'ZipList' -- -- @since 0.6.0 toZipList :: List a -> ZipList a -toZipList = ZipList . K.adapt . toSerial +toZipList = ZipList . ZipSerialM . getSerialT . toSerial diff --git a/src/Streamly/Internal/Data/SmallArray.hs b/src/Streamly/Internal/Data/SmallArray.hs index 23a3a33bba..f66ad34e8a 100644 --- a/src/Streamly/Internal/Data/SmallArray.hs +++ b/src/Streamly/Internal/Data/SmallArray.hs @@ -53,8 +53,7 @@ import Streamly.Internal.Data.SmallArray.Type import Streamly.Internal.Data.Tuple.Strict (Tuple'(..)) import Streamly.Internal.Data.Unfold.Type (Unfold(..)) import Streamly.Internal.Data.Fold.Type (Fold(..)) -import Streamly.Internal.Data.Stream.StreamK.Type (IsStream) -import Streamly.Internal.Data.Stream.Serial (SerialT) +import Streamly.Internal.Data.Stream.Serial (SerialT(..)) import qualified Streamly.Internal.Data.Stream.StreamD as D import qualified Streamly.Internal.Data.Fold.Type as FL @@ -155,17 +154,17 @@ instance NFData a => NFData (SmallArray a) where -- For optimal performance use this with @n@ <= 128. {-# INLINE fromStreamN #-} fromStreamN :: MonadIO m => Int -> SerialT m a -> m (SmallArray a) -fromStreamN n m = do +fromStreamN n (SerialT m) = do when (n < 0) $ error "fromStreamN: negative write count specified" - fromStreamDN n $ D.toStreamD m + fromStreamDN n $ D.fromStreamK m {-# INLINE_EARLY toStream #-} -toStream :: (Monad m, IsStream t) => SmallArray a -> t m a -toStream = D.fromStreamD . toStreamD +toStream :: Monad m => SmallArray a -> SerialT m a +toStream = SerialT . D.toStreamK . toStreamD {-# INLINE_EARLY toStreamRev #-} -toStreamRev :: (Monad m, IsStream t) => SmallArray a -> t m a -toStreamRev = D.fromStreamD . toStreamDRev +toStreamRev :: Monad m => SmallArray a -> SerialT m a +toStreamRev = SerialT . D.toStreamK . toStreamDRev {-# INLINE fold #-} fold :: Monad m => Fold m a b -> SmallArray a -> m b diff --git a/src/Streamly/Internal/Data/Stream/Ahead.hs b/src/Streamly/Internal/Data/Stream/Ahead.hs index aab61fb5ab..3ceaf6310a 100644 --- a/src/Streamly/Internal/Data/Stream/Ahead.hs +++ b/src/Streamly/Internal/Data/Stream/Ahead.hs @@ -22,10 +22,10 @@ -- module Streamly.Internal.Data.Stream.Ahead ( - AheadT + AheadT(..) , Ahead - , fromAhead - , ahead + , aheadK + , consM ) where @@ -52,10 +52,9 @@ import qualified Data.Heap as H import Streamly.Internal.Control.Concurrent (MonadAsync, RunInIO(..), captureMonadState) -import Streamly.Internal.Data.Stream.StreamK.Type - (IsStream(..), Stream, mkStream, foldStream, foldStreamShared) +import Streamly.Internal.Data.Stream.Serial (SerialT(..)) +import Streamly.Internal.Data.Stream.StreamK.Type (Stream) -import qualified Streamly.Internal.Data.Stream.StreamK as K (withLocal) import qualified Streamly.Internal.Data.Stream.StreamK.Type as K import qualified Streamly.Internal.Data.Stream.StreamD.Type as D @@ -321,7 +320,7 @@ processHeap q heap st sv winfo entry sno stopping = loopHeap sno entry let stop = do liftIO (incrementYieldLimit sv) nextHeap seqNo - foldStreamShared st + K.foldStreamShared st (yieldStreamFromHeap seqNo) (singleStreamFromHeap seqNo) stop @@ -380,7 +379,7 @@ processWithoutToken q heap st sv winfo m seqNo = do mrun = runInIO $ svarMrun sv r <- liftIO $ mrun $ - foldStreamShared st + K.foldStreamShared st (\a r -> do runIn <- captureMonadState toHeap $ AheadEntryStream (runIn, K.cons a r)) @@ -451,7 +450,7 @@ processWithToken q heap st sv winfo action sno = do mrun = runInIO $ svarMrun sv r <- liftIO $ mrun $ - foldStreamShared st (yieldOutput sno) (singleOutput sno) stop action + K.foldStreamShared st (yieldOutput sno) (singleOutput sno) stop action res <- restoreM r case res of @@ -479,7 +478,7 @@ processWithToken q heap st sv winfo action sno = do let stop = do liftIO (incrementYieldLimit sv) return $ TokenContinue (seqNo + 1) - foldStreamShared st + K.foldStreamShared st (yieldOutput seqNo) (singleOutput seqNo) stop @@ -513,7 +512,7 @@ processWithToken q heap st sv winfo action sno = do return $ TokenContinue (seqNo + 1) mrun = runInIO $ svarMrun sv r <- liftIO $ mrun $ - foldStreamShared st + K.foldStreamShared st (yieldOutput seqNo) (singleOutput seqNo) stop @@ -604,80 +603,36 @@ workLoopAhead q heap st sv winfo = do -- The only difference between forkSVarAsync and this is that we run the left -- computation without a shared SVar. -forkSVarAhead :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a -forkSVarAhead m1 m2 = mkStream $ \st yld sng stp -> do - sv <- newAheadVar st (concurrently (toStream m1) (toStream m2)) +forkSVarAhead :: MonadAsync m => Stream m a -> Stream m a -> Stream m a +forkSVarAhead m1 m2 = K.mkStream $ \st yld sng stp -> do + sv <- newAheadVar st (concurrently m1 m2) workLoopAhead - foldStream st yld sng stp (fromSVar sv) + K.foldStream st yld sng stp $ getSerialT (fromSVar sv) where - concurrently ma mb = mkStream $ \st yld sng stp -> do + concurrently ma mb = K.mkStream $ \st yld sng stp -> do runInIO <- captureMonadState liftIO $ enqueue (fromJust $ streamVar st) (runInIO, mb) - foldStream st yld sng stp ma + K.foldStream st yld sng stp ma -infixr 6 `ahead` - --- | Appends two streams, both the streams may be evaluated concurrently but --- the outputs are used in the same order as the corresponding actions in the --- original streams, side effects will happen in the order in which the streams --- are evaluated: --- --- >>> import Streamly.Prelude (ahead, SerialT) --- >>> stream1 = Stream.fromEffect (delay 4) :: SerialT IO Int --- >>> stream2 = Stream.fromEffect (delay 2) :: SerialT IO Int --- >>> Stream.toList $ stream1 `ahead` stream2 :: IO [Int] --- 2 sec --- 4 sec --- [4,2] --- --- Multiple streams can be combined. With enough threads, all of them can be --- scheduled simultaneously: --- --- >>> stream3 = Stream.fromEffect (delay 1) --- >>> Stream.toList $ stream1 `ahead` stream2 `ahead` stream3 --- 1 sec --- 2 sec --- 4 sec --- [4,2,1] --- --- With 2 threads, only two can be scheduled at a time, when one of those --- finishes, the third one gets scheduled: --- --- >>> Stream.toList $ Stream.maxThreads 2 $ stream1 `ahead` stream2 `ahead` stream3 --- 2 sec --- 1 sec --- 4 sec --- [4,2,1] --- --- Only streams are scheduled for ahead evaluation, how actions within a stream --- are evaluated depends on the stream type. If it is a concurrent stream they --- will be evaluated concurrently. It may not make much sense combining serial --- streams using 'ahead'. --- --- 'ahead' can be safely used to fold an infinite lazy container of streams. --- --- /Since: 0.3.0 ("Streamly")/ --- --- @since 0.8.0 -{-# INLINE ahead #-} -ahead :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a -ahead m1 m2 = mkStream $ \st yld sng stp -> +{-# INLINE aheadK #-} +aheadK :: MonadAsync m => Stream m a -> Stream m a -> Stream m a +aheadK m1 m2 = K.mkStream $ \st yld sng stp -> case streamVar st of Just sv | svarStyle sv == AheadVar -> do runInIO <- captureMonadState - liftIO $ enqueue sv (runInIO, toStream m2) + liftIO $ enqueue sv (runInIO, m2) -- Always run the left side on a new SVar to avoid complexity in -- sequencing results. This means the left side cannot further -- split into more ahead computations on the same SVar. - foldStream st yld sng stp m1 - _ -> foldStreamShared st yld sng stp (forkSVarAhead m1 m2) + K.foldStream st yld sng stp m1 + _ -> K.foldStreamShared st yld sng stp (forkSVarAhead m1 m2) -- | XXX we can implement it more efficienty by directly implementing instead -- of combining streams using ahead. -{-# INLINE consMAhead #-} -{-# SPECIALIZE consMAhead :: IO a -> AheadT IO a -> AheadT IO a #-} -consMAhead :: MonadAsync m => m a -> AheadT m a -> AheadT m a -consMAhead m r = fromStream $ K.fromEffect m `ahead` toStream r +{-# INLINE consM #-} +{-# SPECIALIZE consM :: IO a -> AheadT IO a -> AheadT IO a #-} +consM :: MonadAsync m => m a -> AheadT m a -> AheadT m a +consM m (AheadT r) = AheadT $ aheadK (K.fromEffect m) r ------------------------------------------------------------------------------ -- AheadT @@ -737,66 +692,58 @@ newtype AheadT m a = AheadT {getAheadT :: Stream m a} -- @since 0.8.0 type Ahead = AheadT IO --- | Fix the type of a polymorphic stream as 'AheadT'. --- --- /Since: 0.3.0 ("Streamly")/ --- --- @since 0.8.0 -fromAhead :: IsStream t => AheadT m a -> t m a -fromAhead = K.adapt - -instance IsStream AheadT where - toStream = getAheadT - fromStream = AheadT - consM = consMAhead - (|:) = consMAhead - ------------------------------------------------------------------------------ -- Semigroup ------------------------------------------------------------------------------ -{-# INLINE mappendAhead #-} -{-# SPECIALIZE mappendAhead :: AheadT IO a -> AheadT IO a -> AheadT IO a #-} -mappendAhead :: MonadAsync m => AheadT m a -> AheadT m a -> AheadT m a -mappendAhead m1 m2 = fromStream $ ahead (toStream m1) (toStream m2) +{-# INLINE append #-} +{-# SPECIALIZE append :: AheadT IO a -> AheadT IO a -> AheadT IO a #-} +append :: MonadAsync m => AheadT m a -> AheadT m a -> AheadT m a +append (AheadT m1) (AheadT m2) = AheadT $ aheadK m1 m2 instance MonadAsync m => Semigroup (AheadT m a) where - (<>) = mappendAhead + (<>) = append ------------------------------------------------------------------------------ -- Monoid ------------------------------------------------------------------------------ instance MonadAsync m => Monoid (AheadT m a) where - mempty = K.nil + mempty = AheadT K.nil mappend = (<>) ------------------------------------------------------------------------------ --- Monad +-- Applicative ------------------------------------------------------------------------------ -{-# INLINE concatMapAhead #-} -{-# SPECIALIZE concatMapAhead :: (a -> AheadT IO b) -> AheadT IO a -> AheadT IO b #-} -concatMapAhead :: MonadAsync m => (a -> AheadT m b) -> AheadT m a -> AheadT m b -concatMapAhead f m = fromStream $ - K.concatMapBy ahead (K.adapt . f) (K.adapt m) - {-# INLINE apAhead #-} apAhead :: MonadAsync m => AheadT m (a -> b) -> AheadT m a -> AheadT m b apAhead (AheadT m1) (AheadT m2) = - let f x1 = K.concatMapBy ahead (pure . x1) m2 - in AheadT $ K.concatMapBy ahead f m1 + let f x1 = K.concatMapWith aheadK (pure . x1) m2 + in AheadT $ K.concatMapWith aheadK f m1 instance (Monad m, MonadAsync m) => Applicative (AheadT m) where {-# INLINE pure #-} pure = AheadT . K.fromPure + {-# INLINE (<*>) #-} (<*>) = apAhead +------------------------------------------------------------------------------ +-- Monad +------------------------------------------------------------------------------ + +{-# INLINE bindAhead #-} +{-# SPECIALIZE bindAhead :: + AheadT IO a -> (a -> AheadT IO b) -> AheadT IO b #-} +bindAhead :: MonadAsync m => AheadT m a -> (a -> AheadT m b) -> AheadT m b +bindAhead (AheadT m) f = AheadT $ K.bindWith aheadK m (getAheadT . f) + instance MonadAsync m => Monad (AheadT m) where return = pure + {-# INLINE (>>=) #-} - (>>=) = flip concatMapAhead + (>>=) = bindAhead ------------------------------------------------------------------------------ -- Other instances diff --git a/src/Streamly/Internal/Data/Stream/Async.hs b/src/Streamly/Internal/Data/Stream/Async.hs index 1b6999f51e..51a55afb51 100644 --- a/src/Streamly/Internal/Data/Stream/Async.hs +++ b/src/Streamly/Internal/Data/Stream/Async.hs @@ -24,18 +24,17 @@ -- module Streamly.Internal.Data.Stream.Async ( - AsyncT + AsyncT(..) , Async - , fromAsync - , async - , (<|) --deprecated - , mkAsync + , consMAsync + , asyncK , mkAsyncK + , mkAsyncD - , WAsyncT + , WAsyncT(..) , WAsync - , fromWAsync - , wAsync + , consMWAsync + , wAsyncK ) where @@ -63,11 +62,10 @@ import Streamly.Internal.Control.Concurrent (MonadAsync, RunInIO(..), captureMonadState) import Streamly.Internal.Data.Atomics (atomicModifyIORefCAS, atomicModifyIORefCAS_) +import Streamly.Internal.Data.Stream.Serial (SerialT (..)) +import Streamly.Internal.Data.Stream.StreamK.Type (Stream) import Streamly.Internal.Data.Stream.SVar.Generate (fromSVar, fromSVarD) -import Streamly.Internal.Data.Stream.StreamK.Type - (IsStream(..), Stream, mkStream, foldStream, adapt, foldStreamShared) -import qualified Streamly.Internal.Data.Stream.StreamK as K (withLocal) import qualified Streamly.Internal.Data.Stream.StreamK.Type as K import qualified Streamly.Internal.Data.Stream.StreamD.Type as D @@ -130,7 +128,7 @@ workLoopLIFO q st sv winfo = run -- monad state here. That way it can work easily for -- distributed case as well. r <- liftIO $ runin $ - foldStreamShared st yieldk single (return Continue) m + K.foldStreamShared st yieldk single (return Continue) m res <- restoreM r case res of Continue -> run @@ -143,7 +141,7 @@ workLoopLIFO q st sv winfo = run yieldk a r = do res <- liftIO $ sendYield sv winfo (ChildYield a) if res - then foldStreamShared st yieldk single (return Continue) r + then K.foldStreamShared st yieldk single (return Continue) r else do runInIO <- captureMonadState liftIO $ enqueueLIFO sv q (runInIO, r) @@ -187,7 +185,7 @@ workLoopLIFOLimited q st sv winfo = run if yieldLimitOk then do r <- liftIO $ runin $ - foldStreamShared st yieldk single incrContinue m + K.foldStreamShared st yieldk single incrContinue m res <- restoreM r case res of Continue -> run @@ -209,7 +207,7 @@ workLoopLIFOLimited q st sv winfo = run res <- liftIO $ sendYield sv winfo (ChildYield a) yieldLimitOk <- liftIO $ decrementYieldLimit sv if res && yieldLimitOk - then foldStreamShared st yieldk single incrContinue r + then K.foldStreamShared st yieldk single incrContinue r else do runInIO <- captureMonadState liftIO $ incrementYieldLimit sv @@ -259,7 +257,7 @@ workLoopFIFO q st sv winfo = run Nothing -> stop Just (RunInIO runin, m) -> do r <- liftIO $ runin $ - foldStreamShared st yieldk single (return Continue) m + K.foldStreamShared st yieldk single (return Continue) m res <- restoreM r case res of Continue -> run @@ -302,7 +300,7 @@ workLoopFIFOLimited q st sv winfo = run if yieldLimitOk then do r <- liftIO $ runin $ - foldStreamShared st yieldk single incrContinue m + K.foldStreamShared st yieldk single incrContinue m res <- restoreM r case res of Continue -> run @@ -545,18 +543,21 @@ newAsyncVar st m = do -- /Pre-release/ -- {-# INLINABLE mkAsyncK #-} -mkAsyncK :: (IsStream t, MonadAsync m) => t m a -> t m a -mkAsyncK m = mkStream $ \st yld sng stp -> do - sv <- newAsyncVar (adaptState st) (toStream m) - foldStream st yld sng stp $ fromSVar sv +mkAsyncK :: MonadAsync m => Stream m a -> Stream m a +mkAsyncK m = K.mkStream $ \st yld sng stp -> do + sv <- newAsyncVar (adaptState st) m + K.foldStream st yld sng stp $ getSerialT $ fromSVar sv +-- +-- This is slightly faster than the CPS version above +-- {-# INLINE_NORMAL mkAsyncD #-} mkAsyncD :: MonadAsync m => D.Stream m a -> D.Stream m a mkAsyncD m = D.Stream step Nothing where step gst Nothing = do - sv <- newAsyncVar gst (D.fromStreamD m) + sv <- newAsyncVar gst (D.toStreamK m) return $ D.Skip $ Just $ fromSVarD sv step gst (Just (D.UnStream step1 st)) = do @@ -566,23 +567,7 @@ mkAsyncD m = D.Stream step Nothing D.Skip s -> D.Skip (Just $ D.Stream step1 s) D.Stop -> D.Stop --- --- This is slightly faster than the CPS version above --- --- | Make the stream producer and consumer run concurrently by introducing a --- buffer between them. The producer thread evaluates the input stream until --- the buffer fills, it terminates if the buffer is full and a worker thread is --- kicked off again to evaluate the remaining stream when there is space in the --- buffer. The consumer consumes the stream lazily from the buffer. --- --- /Since: 0.2.0 (Streamly)/ --- --- @since 0.8.0 --- -{-# INLINE_NORMAL mkAsync #-} -mkAsync :: (K.IsStream t, MonadAsync m) => t m a -> t m a -mkAsync = D.fromStreamD . mkAsyncD . D.toStreamD - +-- XXX We can pass captureMonadState instead of using MonadAsync -- | Create a new SVar and enqueue one stream computation on it. {-# INLINABLE newWAsyncVar #-} newWAsyncVar :: MonadAsync m @@ -590,6 +575,8 @@ newWAsyncVar :: MonadAsync m newWAsyncVar st m = do mrun <- captureMonadState sv <- liftIO $ getFifoSVar st mrun + -- XXX Use just Stream and IO in all the functions below + -- XXX pass mrun instead of calling captureMonadState again inside it sendFirstWorker sv m ------------------------------------------------------------------------------ @@ -656,109 +643,38 @@ newWAsyncVar st m = do -- composition and vice-versa we create a new SVar to isolate the scheduling -- of the two. -forkSVarAsync :: (IsStream t, MonadAsync m) - => SVarStyle -> t m a -> t m a -> t m a -forkSVarAsync style m1 m2 = mkStream $ \st yld sng stp -> do +forkSVarAsync :: MonadAsync m + => SVarStyle -> Stream m a -> Stream m a -> Stream m a +forkSVarAsync style m1 m2 = K.mkStream $ \st yld sng stp -> do sv <- case style of - AsyncVar -> newAsyncVar st (concurrently (toStream m1) (toStream m2)) - WAsyncVar -> newWAsyncVar st (concurrently (toStream m1) (toStream m2)) + AsyncVar -> newAsyncVar st (concurrently m1 m2) + WAsyncVar -> newWAsyncVar st (concurrently m1 m2) _ -> error "illegal svar type" - foldStream st yld sng stp $ fromSVar sv + K.foldStream st yld sng stp $ getSerialT $ fromSVar sv where - concurrently ma mb = mkStream $ \st yld sng stp -> do + concurrently ma mb = K.mkStream $ \st yld sng stp -> do runInIO <- captureMonadState liftIO $ enqueue (fromJust $ streamVar st) (runInIO, mb) - foldStreamShared st yld sng stp ma + K.foldStreamShared st yld sng stp ma {-# INLINE joinStreamVarAsync #-} -joinStreamVarAsync :: (IsStream t, MonadAsync m) - => SVarStyle -> t m a -> t m a -> t m a -joinStreamVarAsync style m1 m2 = mkStream $ \st yld sng stp -> +joinStreamVarAsync :: MonadAsync m + => SVarStyle -> Stream m a -> Stream m a -> Stream m a +joinStreamVarAsync style m1 m2 = K.mkStream $ \st yld sng stp -> case streamVar st of Just sv | svarStyle sv == style -> do runInIO <- captureMonadState - liftIO $ enqueue sv (runInIO, toStream m2) - foldStreamShared st yld sng stp m1 - _ -> foldStreamShared st yld sng stp (forkSVarAsync style m1 m2) + liftIO $ enqueue sv (runInIO, m2) + K.foldStreamShared st yld sng stp m1 + _ -> K.foldStreamShared st yld sng stp (forkSVarAsync style m1 m2) ------------------------------------------------------------------------------ -- Semigroup and Monoid style compositions for parallel actions ------------------------------------------------------------------------------ -infixr 6 `async` - --- | Merges two streams, both the streams may be evaluated concurrently, --- outputs from both are used as they arrive: --- --- >>> import Streamly.Prelude (async) --- >>> stream1 = Stream.fromEffect (delay 4) --- >>> stream2 = Stream.fromEffect (delay 2) --- >>> Stream.toList $ stream1 `async` stream2 --- 2 sec --- 4 sec --- [2,4] --- --- Multiple streams can be combined. With enough threads, all of them can be --- scheduled simultaneously: --- --- >>> stream3 = Stream.fromEffect (delay 1) --- >>> Stream.toList $ stream1 `async` stream2 `async` stream3 --- ... --- [1,2,4] --- --- With 2 threads, only two can be scheduled at a time, when one of those --- finishes, the third one gets scheduled: --- --- >>> Stream.toList $ Stream.maxThreads 2 $ stream1 `async` stream2 `async` stream3 --- ... --- [2,1,4] --- --- With a single thread, it becomes serial: --- --- >>> Stream.toList $ Stream.maxThreads 1 $ stream1 `async` stream2 `async` stream3 --- ... --- [4,2,1] --- --- Only streams are scheduled for async evaluation, how actions within a --- stream are evaluated depends on the stream type. If it is a concurrent --- stream they will be evaluated concurrently. --- --- In the following example, both the streams are scheduled for concurrent --- evaluation but each individual stream is evaluated serially: --- --- >>> stream1 = Stream.fromListM $ Prelude.map delay [3,3] -- SerialT IO Int --- >>> stream2 = Stream.fromListM $ Prelude.map delay [1,1] -- SerialT IO Int --- >>> Stream.toList $ stream1 `async` stream2 -- IO [Int] --- ... --- [1,1,3,3] --- --- If total threads are 2, the third stream is scheduled only after one of the --- first two has finished: --- --- > stream3 = Stream.fromListM $ Prelude.map delay [2,2] -- SerialT IO Int --- > Stream.toList $ Stream.maxThreads 2 $ stream1 `async` stream2 `async` stream3 -- IO [Int] --- ... --- [1,1,3,2,3,2] --- --- Thus 'async' goes deep in first few streams rather than going wide in all --- streams. It prefers to evaluate the leftmost streams as much as possible. --- Because of this behavior, 'async' can be safely used to fold an infinite --- lazy container of streams. --- --- /Since: 0.2.0 ("Streamly")/ --- --- @since 0.8.0 -{-# INLINE async #-} -async :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a -async = joinStreamVarAsync AsyncVar - --- | Same as 'async'. --- --- @since 0.1.0 -{-# DEPRECATED (<|) "Please use 'async' instead." #-} -{-# INLINE (<|) #-} -(<|) :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a -(<|) = async +{-# INLINE asyncK #-} +asyncK :: MonadAsync m => Stream m a -> Stream m a -> Stream m a +asyncK = joinStreamVarAsync AsyncVar -- IMPORTANT: using a monomorphically typed and SPECIALIZED consMAsync makes a -- huge difference in the performance of consM in IsStream instance even we @@ -769,7 +685,7 @@ async = joinStreamVarAsync AsyncVar {-# INLINE consMAsync #-} {-# SPECIALIZE consMAsync :: IO a -> AsyncT IO a -> AsyncT IO a #-} consMAsync :: MonadAsync m => m a -> AsyncT m a -> AsyncT m a -consMAsync m r = fromStream $ K.fromEffect m `async` toStream r +consMAsync m (AsyncT r) = AsyncT $ asyncK (K.fromEffect m) r ------------------------------------------------------------------------------ -- AsyncT @@ -829,40 +745,26 @@ newtype AsyncT m a = AsyncT {getAsyncT :: Stream m a} -- @since 0.8.0 type Async = AsyncT IO --- | Fix the type of a polymorphic stream as 'AsyncT'. --- --- /Since: 0.1.0 ("Streamly")/ --- --- @since 0.8.0 -fromAsync :: IsStream t => AsyncT m a -> t m a -fromAsync = adapt - -instance IsStream AsyncT where - toStream = getAsyncT - fromStream = AsyncT - consM = consMAsync - (|:) = consMAsync - ------------------------------------------------------------------------------ -- Semigroup ------------------------------------------------------------------------------ -- Monomorphically typed version of "async" for better performance of Semigroup -- instance. -{-# INLINE mappendAsync #-} -{-# SPECIALIZE mappendAsync :: AsyncT IO a -> AsyncT IO a -> AsyncT IO a #-} -mappendAsync :: MonadAsync m => AsyncT m a -> AsyncT m a -> AsyncT m a -mappendAsync m1 m2 = fromStream $ async (toStream m1) (toStream m2) +{-# INLINE append #-} +{-# SPECIALIZE append :: AsyncT IO a -> AsyncT IO a -> AsyncT IO a #-} +append :: MonadAsync m => AsyncT m a -> AsyncT m a -> AsyncT m a +append (AsyncT m1) (AsyncT m2) = AsyncT $ asyncK m1 m2 instance MonadAsync m => Semigroup (AsyncT m a) where - (<>) = mappendAsync + (<>) = append ------------------------------------------------------------------------------ -- Monoid ------------------------------------------------------------------------------ instance MonadAsync m => Monoid (AsyncT m a) where - mempty = K.nil + mempty = AsyncT K.nil mappend = (<>) ------------------------------------------------------------------------------ @@ -873,12 +775,13 @@ instance MonadAsync m => Monoid (AsyncT m a) where {-# SPECIALIZE apAsync :: AsyncT IO (a -> b) -> AsyncT IO a -> AsyncT IO b #-} apAsync :: MonadAsync m => AsyncT m (a -> b) -> AsyncT m a -> AsyncT m b apAsync (AsyncT m1) (AsyncT m2) = - let f x1 = K.concatMapBy async (pure . x1) m2 - in AsyncT $ K.concatMapBy async f m1 + let f x1 = K.concatMapWith asyncK (pure . x1) m2 + in AsyncT $ K.concatMapWith asyncK f m1 instance (Monad m, MonadAsync m) => Applicative (AsyncT m) where {-# INLINE pure #-} pure = AsyncT . K.fromPure + {-# INLINE (<*>) #-} (<*>) = apAsync @@ -889,9 +792,10 @@ instance (Monad m, MonadAsync m) => Applicative (AsyncT m) where -- GHC: if we change the implementation of bindWith with arguments in a -- different order we see a significant performance degradation (~2x). {-# INLINE bindAsync #-} -{-# SPECIALIZE bindAsync :: AsyncT IO a -> (a -> AsyncT IO b) -> AsyncT IO b #-} +{-# SPECIALIZE bindAsync :: + AsyncT IO a -> (a -> AsyncT IO b) -> AsyncT IO b #-} bindAsync :: MonadAsync m => AsyncT m a -> (a -> AsyncT m b) -> AsyncT m b -bindAsync m f = fromStream $ K.bindWith async (adapt m) (adapt . f) +bindAsync (AsyncT m) f = AsyncT $ K.bindWith asyncK m (getAsyncT . f) -- GHC: if we specify arguments in the definition of (>>=) we see a significant -- performance degradation (~2x). @@ -909,84 +813,16 @@ MONAD_COMMON_INSTANCES(AsyncT, MONADPARALLEL) -- WAsyncT ------------------------------------------------------------------------------ +{-# INLINE wAsyncK #-} +wAsyncK :: MonadAsync m => Stream m a -> Stream m a -> Stream m a +wAsyncK = joinStreamVarAsync WAsyncVar + -- | XXX we can implement it more efficienty by directly implementing instead -- of combining streams using wAsync. {-# INLINE consMWAsync #-} {-# SPECIALIZE consMWAsync :: IO a -> WAsyncT IO a -> WAsyncT IO a #-} consMWAsync :: MonadAsync m => m a -> WAsyncT m a -> WAsyncT m a -consMWAsync m r = fromStream $ K.fromEffect m `wAsync` toStream r - -infixr 6 `wAsync` - --- | For singleton streams, 'wAsync' is the same as 'async'. See 'async' for --- singleton stream behavior. For multi-element streams, while 'async' is left --- biased i.e. it tries to evaluate the left side stream as much as possible, --- 'wAsync' tries to schedule them both fairly. In other words, 'async' goes --- deep while 'wAsync' goes wide. However, outputs are always used as they --- arrive. --- --- With a single thread, 'async' starts behaving like 'serial' while 'wAsync' --- starts behaving like 'wSerial'. --- --- >>> import Streamly.Prelude (wAsync) --- >>> stream1 = Stream.fromList [1,2,3] --- >>> stream2 = Stream.fromList [4,5,6] --- >>> Stream.toList $ Stream.fromAsync $ Stream.maxThreads 1 $ stream1 `async` stream2 --- [1,2,3,4,5,6] --- --- >>> Stream.toList $ Stream.fromWAsync $ Stream.maxThreads 1 $ stream1 `wAsync` stream2 --- [1,4,2,5,3,6] --- --- With two threads available, and combining three streams: --- --- >>> stream3 = Stream.fromList [7,8,9] --- >>> Stream.toList $ Stream.fromAsync $ Stream.maxThreads 2 $ stream1 `async` stream2 `async` stream3 --- [1,2,3,4,5,6,7,8,9] --- --- >>> Stream.toList $ Stream.fromWAsync $ Stream.maxThreads 2 $ stream1 `wAsync` stream2 `wAsync` stream3 --- [1,4,2,7,5,3,8,6,9] --- --- This operation cannot be used to fold an infinite lazy container of streams, --- because it schedules all the streams in a round robin manner. --- --- Note that 'WSerialT' and single threaded 'WAsyncT' both interleave streams --- but the exact scheduling is slightly different in both cases. --- --- @since 0.8.0 --- --- /Since: 0.2.0 ("Streamly")/ - --- Scheduling details: --- --- This is how the execution of the above example proceeds: --- --- 1. The scheduler queue is initialized with @[S.fromList [1,2,3], --- (S.fromList [4,5,6]) \<> (S.fromList [7,8,9])]@ assuming the head of the --- queue is represented by the rightmost item. --- 2. @S.fromList [1,2,3]@ is executed, yielding the element @1@ and putting --- @[2,3]@ at the back of the scheduler queue. The scheduler queue now looks --- like @[(S.fromList [4,5,6]) \<> (S.fromList [7,8,9]), S.fromList [2,3]]@. --- 3. Now @(S.fromList [4,5,6]) \<> (S.fromList [7,8,9])@ is picked up for --- execution, @S.fromList [7,8,9]@ is added at the back of the queue and --- @S.fromList [4,5,6]@ is executed, yielding the element @4@ and adding --- @S.fromList [5,6]@ at the back of the queue. The queue now looks like --- @[S.fromList [2,3], S.fromList [7,8,9], S.fromList [5,6]]@. --- 4. Note that the scheduler queue expands by one more stream component in --- every pass because one more @<>@ is broken down into two components. At this --- point there are no more @<>@ operations to be broken down further and the --- queue has reached its maximum size. Now these streams are scheduled in --- round-robin fashion yielding @[2,7,5,3,8,6,9]@. --- --- As we see above, in a right associated expression composed with @<>@, only --- one @<>@ operation is broken down into two components in one execution, --- therefore, if we have @n@ streams composed using @<>@ it will take @n@ --- scheduler passes to expand the whole expression. By the time @n-th@ --- component is added to the scheduler queue, the first component would have --- received @n@ scheduler passes. --- -{-# INLINE wAsync #-} -wAsync :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a -wAsync = joinStreamVarAsync WAsyncVar +consMWAsync m (WAsyncT r) = WAsyncT $ wAsyncK (K.fromEffect m) r -- | For 'WAsyncT' streams: -- @@ -1132,38 +968,24 @@ newtype WAsyncT m a = WAsyncT {getWAsyncT :: Stream m a} -- @since 0.8.0 type WAsync = WAsyncT IO --- | Fix the type of a polymorphic stream as 'WAsyncT'. --- --- /Since: 0.2.0 ("Streamly")/ --- --- @since 0.8.0 -fromWAsync :: IsStream t => WAsyncT m a -> t m a -fromWAsync = adapt - -instance IsStream WAsyncT where - toStream = getWAsyncT - fromStream = WAsyncT - consM = consMWAsync - (|:) = consMWAsync - ------------------------------------------------------------------------------ -- Semigroup ------------------------------------------------------------------------------ -{-# INLINE mappendWAsync #-} -{-# SPECIALIZE mappendWAsync :: WAsyncT IO a -> WAsyncT IO a -> WAsyncT IO a #-} -mappendWAsync :: MonadAsync m => WAsyncT m a -> WAsyncT m a -> WAsyncT m a -mappendWAsync m1 m2 = fromStream $ wAsync (toStream m1) (toStream m2) +{-# INLINE wAppend #-} +{-# SPECIALIZE wAppend :: WAsyncT IO a -> WAsyncT IO a -> WAsyncT IO a #-} +wAppend :: MonadAsync m => WAsyncT m a -> WAsyncT m a -> WAsyncT m a +wAppend (WAsyncT m1) (WAsyncT m2) = WAsyncT $ wAsyncK m1 m2 instance MonadAsync m => Semigroup (WAsyncT m a) where - (<>) = mappendWAsync + (<>) = wAppend ------------------------------------------------------------------------------ -- Monoid ------------------------------------------------------------------------------ instance MonadAsync m => Monoid (WAsyncT m a) where - mempty = K.nil + mempty = WAsyncT K.nil mappend = (<>) ------------------------------------------------------------------------------ @@ -1171,11 +993,12 @@ instance MonadAsync m => Monoid (WAsyncT m a) where ------------------------------------------------------------------------------ {-# INLINE apWAsync #-} -{-# SPECIALIZE apWAsync :: WAsyncT IO (a -> b) -> WAsyncT IO a -> WAsyncT IO b #-} +{-# SPECIALIZE apWAsync :: + WAsyncT IO (a -> b) -> WAsyncT IO a -> WAsyncT IO b #-} apWAsync :: MonadAsync m => WAsyncT m (a -> b) -> WAsyncT m a -> WAsyncT m b apWAsync (WAsyncT m1) (WAsyncT m2) = - let f x1 = K.concatMapBy wAsync (pure . x1) m2 - in WAsyncT $ K.concatMapBy wAsync f m1 + let f x1 = K.concatMapWith wAsyncK (pure . x1) m2 + in WAsyncT $ K.concatMapWith wAsyncK f m1 -- GHC: if we specify arguments in the definition of (<*>) we see a significant -- performance degradation (~2x). @@ -1190,9 +1013,10 @@ instance (Monad m, MonadAsync m) => Applicative (WAsyncT m) where -- GHC: if we change the implementation of bindWith with arguments in a -- different order we see a significant performance degradation (~2x). {-# INLINE bindWAsync #-} -{-# SPECIALIZE bindWAsync :: WAsyncT IO a -> (a -> WAsyncT IO b) -> WAsyncT IO b #-} +{-# SPECIALIZE bindWAsync :: + WAsyncT IO a -> (a -> WAsyncT IO b) -> WAsyncT IO b #-} bindWAsync :: MonadAsync m => WAsyncT m a -> (a -> WAsyncT m b) -> WAsyncT m b -bindWAsync m f = fromStream $ K.bindWith wAsync (adapt m) (adapt . f) +bindWAsync (WAsyncT m) f = WAsyncT $ K.bindWith wAsyncK m (getWAsyncT . f) -- GHC: if we specify arguments in the definition of (>>=) we see a significant -- performance degradation (~2x). diff --git a/src/Streamly/Internal/Data/Stream/Instances.hs b/src/Streamly/Internal/Data/Stream/Instances.hs index d8fd5244df..b77ce68632 100644 --- a/src/Streamly/Internal/Data/Stream/Instances.hs +++ b/src/Streamly/Internal/Data/Stream/Instances.hs @@ -10,7 +10,8 @@ #define MONAD_COMMON_INSTANCES(STREAM,CONSTRAINT) \ instance Monad m => Functor (STREAM m) where { \ {-# INLINE fmap #-}; \ - fmap f (STREAM m) = D.fromStreamD $ D.mapM (return . f) $ D.toStreamD m; \ + fmap f (STREAM m) = \ + STREAM $ D.toStreamK $ D.mapM (return . f) $ D.fromStreamK m; \ {-# INLINE (<$) #-}; \ (<$) = fmap . const }; \ \ @@ -32,7 +33,7 @@ instance (MonadError e m CONSTRAINT) => MonadError e (STREAM m) where { \ \ instance (MonadReader r m CONSTRAINT) => MonadReader r (STREAM m) where { \ ask = lift ask; \ - local f m = fromStream $ K.withLocal f (toStream m) }; \ + local f (STREAM m) = STREAM $ K.withLocal f m }; \ \ instance (MonadState s m CONSTRAINT) => MonadState s (STREAM m) where { \ {-# INLINE get #-}; \ @@ -59,7 +60,7 @@ instance (MonadState s m CONSTRAINT) => MonadState s (STREAM m) where { \ #define NFDATA1_INSTANCE(STREAM) \ instance NFData1 (STREAM Identity) where { \ {-# INLINE liftRnf #-}; \ - liftRnf r = runIdentity . P.foldl' (\_ x -> r x) () } + liftRnf f (STREAM xs) = runIdentity $ P.foldl' (\_ x -> f x) () xs} #else #define NFDATA1_INSTANCE(STREAM) #endif @@ -68,17 +69,17 @@ instance NFData1 (STREAM Identity) where { \ instance IsList (STREAM Identity a) where { \ type (Item (STREAM Identity a)) = a; \ {-# INLINE fromList #-}; \ - fromList = P.fromList; \ + fromList xs = STREAM $ P.fromList xs; \ {-# INLINE toList #-}; \ - toList = runIdentity . P.toList }; \ + toList (STREAM xs) = runIdentity $ P.toList xs }; \ \ instance Eq a => Eq (STREAM Identity a) where { \ {-# INLINE (==) #-}; \ - (==) xs ys = runIdentity $ P.eqBy (==) xs ys }; \ + (==) (STREAM xs) (STREAM ys) = runIdentity $ P.eqBy (==) xs ys }; \ \ instance Ord a => Ord (STREAM Identity a) where { \ {-# INLINE compare #-}; \ - compare xs ys = runIdentity $ P.cmpBy compare xs ys; \ + compare (STREAM xs) (STREAM ys) = runIdentity $ P.cmpBy compare xs ys; \ {-# INLINE (<) #-}; \ x < y = case compare x y of { LT -> True; _ -> False }; \ {-# INLINE (<=) #-}; \ @@ -104,11 +105,11 @@ instance Read a => Read (STREAM Identity a) where { \ \ instance (a ~ Char) => IsString (STREAM Identity a) where { \ {-# INLINE fromString #-}; \ - fromString = P.fromList }; \ + fromString xs = STREAM $ P.fromList xs }; \ \ instance NFData a => NFData (STREAM Identity a) where { \ {-# INLINE rnf #-}; \ - rnf = runIdentity . P.foldl' (\_ x -> rnf x) () }; \ + rnf (STREAM xs) = runIdentity $ P.foldl' (\_ x -> rnf x) () xs}; \ ------------------------------------------------------------------------------- -- Foldable @@ -125,7 +126,7 @@ instance NFData a => NFData (STREAM Identity a) where { \ instance (Foldable m, Monad m) => Foldable (STREAM m) where { \ \ {-# INLINE foldMap #-}; \ - foldMap f = fold . P.foldr (mappend . f) mempty; \ + foldMap f (STREAM xs) = fold $ P.foldr (mappend . f) mempty xs; \ \ {-# INLINE foldr #-}; \ foldr f z t = appEndo (foldMap (Endo #. f) t) z; \ @@ -169,5 +170,6 @@ instance (Foldable m, Monad m) => Foldable (STREAM m) where { \ #define TRAVERSABLE_INSTANCE(STREAM) \ instance Traversable (STREAM Identity) where { \ {-# INLINE traverse #-}; \ - traverse f s = runIdentity $ P.foldr consA (pure mempty) s \ + traverse f (STREAM xs) = \ + fmap STREAM $ runIdentity $ P.foldr consA (pure mempty) xs \ where { consA x ys = liftA2 K.cons (f x) ys }} diff --git a/src/Streamly/Internal/Data/Stream/IsStream.hs b/src/Streamly/Internal/Data/Stream/IsStream.hs index af50f202c9..217ad7dca7 100644 --- a/src/Streamly/Internal/Data/Stream/IsStream.hs +++ b/src/Streamly/Internal/Data/Stream/IsStream.hs @@ -11,7 +11,7 @@ -- experimental APIs. module Streamly.Internal.Data.Stream.IsStream - ( module Streamly.Internal.Data.Stream.IsStream.Types + ( module Streamly.Internal.Data.Stream.IsStream.Type , module Streamly.Internal.Data.Stream.IsStream.Generate , module Streamly.Internal.Data.Stream.IsStream.Eliminate , module Streamly.Internal.Data.Stream.IsStream.Transform @@ -31,4 +31,6 @@ import Streamly.Internal.Data.Stream.IsStream.Lift import Streamly.Internal.Data.Stream.IsStream.Expand import Streamly.Internal.Data.Stream.IsStream.Reduce import Streamly.Internal.Data.Stream.IsStream.Transform -import Streamly.Internal.Data.Stream.IsStream.Types +import Streamly.Internal.Data.Stream.IsStream.Type + hiding (cmpBy, drain, eqBy, foldl', foldrM, foldr, fold, toList, toStream + , fromEffect, fromPure, repeat) diff --git a/src/Streamly/Internal/Data/Stream/IsStream/Combinators.hs b/src/Streamly/Internal/Data/Stream/IsStream/Combinators.hs index f82f1b0175..fa93aac41e 100644 --- a/src/Streamly/Internal/Data/Stream/IsStream/Combinators.hs +++ b/src/Streamly/Internal/Data/Stream/IsStream/Combinators.hs @@ -26,9 +26,12 @@ where import Control.Monad.IO.Class (MonadIO(liftIO)) import Data.Int (Int64) -import Streamly.Internal.Data.SVar -import Streamly.Internal.Data.Stream.StreamK.Type +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream, mkStream, foldStreamShared) import Streamly.Internal.Data.Stream.Serial (SerialT) +import Streamly.Internal.Data.Stream.StreamK.Type (Stream) + +import Streamly.Internal.Data.SVar ------------------------------------------------------------------------------- -- Concurrency control diff --git a/src/Streamly/Internal/Data/Stream/IsStream/Common.hs b/src/Streamly/Internal/Data/Stream/IsStream/Common.hs index 4828cc5490..81e5509cca 100644 --- a/src/Streamly/Internal/Data/Stream/IsStream/Common.hs +++ b/src/Streamly/Internal/Data/Stream/IsStream/Common.hs @@ -26,6 +26,7 @@ module Streamly.Internal.Data.Stream.IsStream.Common , fold_ -- * Transformation + , map , scanlMAfter' , postscanlM' , smapM @@ -39,12 +40,21 @@ module Streamly.Internal.Data.Stream.IsStream.Common , reverse , reverse' + -- * Concurrent + , mkAsync + , mkParallel + , parallelFst + -- * Nesting , concatM , concatMapM , concatMap , splitOnSeq + -- * Zipping + , zipWithM + , zipWith + -- * Deprecated , yield , yieldM @@ -60,15 +70,15 @@ import Streamly.Internal.Control.Concurrent (MonadAsync) import Streamly.Internal.Data.Array.Foreign.Type (Array) import Streamly.Internal.Data.Fold.Type (Fold (..)) import Streamly.Internal.Data.Stream.IsStream.Combinators (maxYields) -import Streamly.Internal.Data.Stream.Prelude (fromStreamS, toStreamS) +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream(..), fromStreamD, toStreamD, fromStreamS, toStreamS) import Streamly.Internal.Data.Stream.Serial (SerialT) -import Streamly.Internal.Data.Stream.StreamD.Type (fromStreamD, toStreamD) -import Streamly.Internal.Data.Stream.StreamK.Type (IsStream()) import Streamly.Internal.Data.Time.Units (AbsTime, RelTime64, addToAbsTime64) +import qualified Streamly.Internal.Data.Array.Foreign.Type as A +import qualified Streamly.Internal.Data.Stream.Async as Async +import qualified Streamly.Internal.Data.Stream.IsStream.Type as IsStream import qualified Streamly.Internal.Data.Stream.Parallel as Par -import qualified Streamly.Internal.Data.Stream.Serial as Serial -import qualified Streamly.Internal.Data.Stream.StreamK as K (repeatM) import qualified Streamly.Internal.Data.Stream.StreamK.Type as K import qualified Streamly.Internal.Data.Stream.StreamD as D #ifdef USE_STREAMK_ONLY @@ -76,8 +86,9 @@ import qualified Streamly.Internal.Data.Stream.StreamK as S #else import qualified Streamly.Internal.Data.Stream.StreamD as S #endif +import Streamly.Internal.System.IO (defaultChunkSize) -import Prelude hiding (take, takeWhile, drop, reverse, concatMap) +import Prelude hiding (take, takeWhile, drop, reverse, concatMap, map, zipWith) -- -- $setup @@ -124,7 +135,7 @@ import Prelude hiding (take, takeWhile, drop, reverse, concatMap) -- {-# INLINE fromPure #-} fromPure :: IsStream t => a -> t m a -fromPure = K.fromPure +fromPure = fromStream . K.fromPure -- | Same as 'fromPure' -- @@ -151,7 +162,7 @@ yield = fromPure -- {-# INLINE fromEffect #-} fromEffect :: (Monad m, IsStream t) => m a -> t m a -fromEffect = K.fromEffect +fromEffect = fromStream . K.fromEffect -- | Same as 'fromEffect' -- @@ -179,7 +190,7 @@ yieldM = fromEffect -- @since 0.2.0 {-# INLINE_EARLY repeatM #-} repeatM :: (IsStream t, MonadAsync m) => m a -> t m a -repeatM = K.repeatM +repeatM = K.repeatMWith IsStream.consM {-# RULES "repeatM serial" repeatM = repeatMSerial #-} {-# INLINE repeatMSerial #-} @@ -286,13 +297,30 @@ fold fl strm = do {-# INLINE fold_ #-} fold_ :: Monad m => Fold m a b -> SerialT m a -> m (b, SerialT m a) fold_ fl strm = do - (b, str) <- D.fold_ fl $ D.toStreamD strm - return $! (b, D.fromStreamD str) + (b, str) <- D.fold_ fl $ IsStream.toStreamD strm + return $! (b, IsStream.fromStreamD str) ------------------------------------------------------------------------------ -- Transformation ------------------------------------------------------------------------------ +-- | +-- @ +-- map = fmap +-- @ +-- +-- Same as 'fmap'. +-- +-- @ +-- > S.toList $ S.map (+1) $ S.fromList [1,2,3] +-- [2,3,4] +-- @ +-- +-- @since 0.4.0 +{-# INLINE map #-} +map :: (IsStream t, Monad m) => (a -> b) -> t m a -> t m b +map f = fromStreamD . D.map f . toStreamD + -- | @scanlMAfter' accumulate initial done stream@ is like 'scanlM'' except -- that it provides an additional @done@ function to be applied on the -- accumulator when the stream stops. The result of @done@ is also emitted in @@ -366,7 +394,7 @@ smapM step initial stream = (\(s, _) a -> step s a) (fmap (,undefined) initial) stream - in Serial.map snd r + in map snd r ------------------------------------------------------------------------------ -- Transformation - Trimming @@ -439,7 +467,7 @@ intersperseM m = fromStreamS . S.intersperseM m . toStreamS interjectSuffix :: (IsStream t, MonadAsync m) => Double -> m a -> t m a -> t m a -interjectSuffix n f xs = xs `Par.parallelFst` repeatM timed +interjectSuffix n f xs = xs `parallelFst` repeatM timed where timed = liftIO (threadDelay (round $ n * 1000000)) >> f ------------------------------------------------------------------------------ @@ -469,7 +497,59 @@ reverse s = fromStreamS $ S.reverse $ toStreamS s -- /Pre-release/ {-# INLINE reverse' #-} reverse' :: (IsStream t, MonadIO m, Storable a) => t m a -> t m a -reverse' s = fromStreamD $ D.reverse' $ toStreamD s +-- reverse' s = fromStreamD $ D.reverse' $ toStreamD s +reverse' = + fromStreamD + . A.flattenArraysRev -- unfoldMany A.readRev + . D.fromStreamK + . K.reverse + . D.toStreamK + . A.arraysOf defaultChunkSize + . toStreamD + +------------------------------------------------------------------------------ +-- Concurrent Transformations and Combining +------------------------------------------------------------------------------ + +-- | Make the stream producer and consumer run concurrently by introducing a +-- buffer between them. The producer thread evaluates the input stream until +-- the buffer fills, it terminates if the buffer is full and a worker thread is +-- kicked off again to evaluate the remaining stream when there is space in the +-- buffer. The consumer consumes the stream lazily from the buffer. +-- +-- /Since: 0.2.0 (Streamly)/ +-- +-- @since 0.8.0 +-- +{-# INLINE_NORMAL mkAsync #-} +mkAsync :: (IsStream t, MonadAsync m) => t m a -> t m a +mkAsync = fromStreamD . Async.mkAsyncD . toStreamD + +-- Compare with mkAsync. mkAsync uses an Async style SVar whereas this uses a +-- parallel style SVar for evaluation. Currently, parallel style cannot use +-- rate control whereas Async style can use rate control. In async style SVar +-- the worker thread terminates when the buffer is full whereas in Parallel +-- style it blocks. +-- +-- | Make the stream producer and consumer run concurrently by introducing a +-- buffer between them. The producer thread evaluates the input stream until +-- the buffer fills, it blocks if the buffer is full until there is space in +-- the buffer. The consumer consumes the stream lazily from the buffer. +-- +-- @mkParallel = IsStream.fromStreamD . mkParallelD . IsStream.toStreamD@ +-- +-- /Pre-release/ +-- +{-# INLINE_NORMAL mkParallel #-} +mkParallel :: (IsStream t, MonadAsync m) => t m a -> t m a +mkParallel = fromStreamD . Par.mkParallelD . toStreamD + +-- | Like `parallel` but stops the output as soon as the first stream stops. +-- +-- /Pre-release/ +{-# INLINE parallelFst #-} +parallelFst :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a +parallelFst m1 m2 = fromStream $ Par.parallelFstK (toStream m1) (toStream m2) ------------------------------------------------------------------------------ -- Combine streams and flatten @@ -575,4 +655,37 @@ concatM generator = concatMapM (\() -> generator) (fromPure ()) splitOnSeq :: (IsStream t, MonadIO m, Storable a, Enum a, Eq a) => Array a -> Fold m a b -> t m a -> t m b -splitOnSeq patt f m = D.fromStreamD $ D.splitOnSeq patt f (D.toStreamD m) +splitOnSeq patt f m = + IsStream.fromStreamD $ D.splitOnSeq patt f (IsStream.toStreamD m) + +------------------------------------------------------------------------------ +-- Zipping +------------------------------------------------------------------------------ + +-- | Like 'zipWith' but using a monadic zipping function. +-- +-- @since 0.4.0 +{-# INLINABLE zipWithM #-} +zipWithM :: (IsStream t, Monad m) => (a -> b -> m c) -> t m a -> t m b -> t m c +zipWithM f m1 m2 = + IsStream.fromStreamS + $ S.zipWithM f (IsStream.toStreamS m1) (IsStream.toStreamS m2) + +-- | Stream @a@ is evaluated first, followed by stream @b@, the resulting +-- elements @a@ and @b@ are then zipped using the supplied zip function and the +-- result @c@ is yielded to the consumer. +-- +-- If stream @a@ or stream @b@ ends, the zipped stream ends. If stream @b@ ends +-- first, the element @a@ from previous evaluation of stream @a@ is discarded. +-- +-- @ +-- > S.toList $ S.zipWith (+) (S.fromList [1,2,3]) (S.fromList [4,5,6]) +-- [5,7,9] +-- @ +-- +-- @since 0.1.0 +{-# INLINABLE zipWith #-} +zipWith :: (IsStream t, Monad m) => (a -> b -> c) -> t m a -> t m b -> t m c +zipWith f m1 m2 = + IsStream.fromStreamS + $ S.zipWith f (IsStream.toStreamS m1) (IsStream.toStreamS m2) diff --git a/src/Streamly/Internal/Data/Stream/IsStream/Eliminate.hs b/src/Streamly/Internal/Data/Stream/IsStream/Eliminate.hs index 7c6590a6ea..24f45c0370 100644 --- a/src/Streamly/Internal/Data/Stream/IsStream/Eliminate.hs +++ b/src/Streamly/Internal/Data/Stream/IsStream/Eliminate.hs @@ -45,6 +45,10 @@ module Streamly.Internal.Data.Stream.IsStream.Eliminate , foldr -- * Left Folds + -- Lazy left folds are useful only for reversing the stream + , foldlS + , foldlT + , foldl' , foldl1' , foldlM' @@ -151,24 +155,23 @@ where import Control.Monad.Catch (MonadThrow) import Control.Monad.IO.Class (MonadIO(..)) +import Control.Monad.Trans.Class (MonadTrans(..)) import Foreign.Storable (Storable) import Streamly.Internal.Control.Concurrent (MonadAsync) import Streamly.Internal.Data.Parser (Parser (..)) import Streamly.Internal.Data.SVar (defState) import Streamly.Internal.Data.Stream.IsStream.Common ( fold, fold_, drop, findIndices, reverse, splitOnSeq, take - , takeWhile) -import Streamly.Internal.Data.Stream.Prelude (toStreamS) -import Streamly.Internal.Data.Stream.StreamD (fromStreamD, toStreamD) -import Streamly.Internal.Data.Stream.StreamK (IsStream) -import Streamly.Internal.Data.Stream.Serial (SerialT) + , takeWhile, mkParallel) +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream, toStreamS, fromStreamD, toStreamD) +import Streamly.Internal.Data.Stream.Serial (SerialT(..)) import qualified Streamly.Internal.Data.Array.Foreign.Type as A import qualified Streamly.Internal.Data.Fold as FL -import qualified Streamly.Internal.Data.Stream.Parallel as Par -import qualified Streamly.Internal.Data.Stream.Prelude as P +import qualified Streamly.Internal.Data.Stream.IsStream.Type as IsStream import qualified Streamly.Internal.Data.Stream.StreamD as D -import qualified Streamly.Internal.Data.Stream.StreamK as K +import qualified Streamly.Internal.Data.Stream.StreamK.Type as K import qualified Streamly.Internal.Data.Parser.ParserD as PRD import qualified Streamly.Internal.Data.Parser.ParserK.Type as PRK import qualified System.IO as IO @@ -212,7 +215,7 @@ import Prelude hiding -- @since 0.1.0 {-# INLINE uncons #-} uncons :: (IsStream t, Monad m) => SerialT m a -> m (Maybe (a, t m a)) -uncons m = K.uncons (K.adapt m) +uncons (SerialT m) = fmap (fmap (fmap IsStream.fromStream)) $ K.uncons m ------------------------------------------------------------------------------ -- Right Folds @@ -240,7 +243,7 @@ uncons m = K.uncons (K.adapt m) -- /Since: 0.1.0/ {-# INLINE foldrM #-} foldrM :: Monad m => (a -> m b -> m b) -> m b -> SerialT m a -> m b -foldrM = P.foldrM +foldrM = IsStream.foldrM -- | Right fold, lazy for lazy monads and pure streams, and strict for strict -- monads. @@ -255,7 +258,7 @@ foldrM = P.foldrM -- @since 0.1.0 {-# INLINE foldr #-} foldr :: Monad m => (a -> b -> b) -> b -> SerialT m a -> m b -foldr = P.foldr +foldr = IsStream.foldr -- XXX This seems to be of limited use as it cannot be used to construct -- recursive structures and for reduction foldl1' is better. @@ -273,6 +276,27 @@ foldr1 f m = S.foldr1 f (toStreamS m) -- Left Folds ------------------------------------------------------------------------------ +-- | Lazy left fold to a stream. +{-# INLINE foldlS #-} +foldlS :: IsStream t => (t m b -> a -> t m b) -> t m b -> t m a -> t m b +foldlS f z = + IsStream.fromStream + . K.foldlS + (\xs x -> IsStream.toStream $ f (IsStream.fromStream xs) x) + (IsStream.toStream z) + . IsStream.toStream + +-- | Lazy left fold to a transformer monad. +-- +-- For example, to reverse a stream: +-- +-- > S.toList $ S.foldlT (flip S.cons) S.nil $ (S.fromList [1..5] :: SerialT IO Int) +-- +{-# INLINE foldlT #-} +foldlT :: (Monad m, IsStream t, Monad (s m), MonadTrans s) + => (s m b -> a -> s m b) -> s m b -> t m a -> s m b +foldlT f z s = S.foldlT f z (toStreamS s) + -- | Strict left fold with an extraction function. Like the standard strict -- left fold, but applies a user supplied extraction function (the third -- argument) to the folded value at the end. This is designed to work with the @@ -282,7 +306,7 @@ foldr1 f m = S.foldr1 f (toStreamS m) {-# DEPRECATED foldx "Please use foldl' followed by fmap instead." #-} {-# INLINE foldx #-} foldx :: Monad m => (x -> a -> x) -> x -> (x -> b) -> SerialT m a -> m b -foldx = P.foldlx' +foldx = IsStream.foldlx' -- | Left associative/strict push fold. @foldl' reduce initial stream@ invokes -- @reduce@ with the accumulator and the next input in the input stream, using @@ -295,7 +319,7 @@ foldx = P.foldlx' -- @since 0.2.0 {-# INLINE foldl' #-} foldl' :: Monad m => (b -> a -> b) -> b -> SerialT m a -> m b -foldl' = P.foldl' +foldl' = IsStream.foldl' -- | Strict left fold, for non-empty streams, using first element as the -- starting value. Returns 'Nothing' if the stream is empty. @@ -317,7 +341,7 @@ foldl1' step m = do {-# DEPRECATED foldxM "Please use foldlM' followed by fmap instead." #-} {-# INLINE foldxM #-} foldxM :: Monad m => (x -> a -> m x) -> m x -> (x -> m b) -> SerialT m a -> m b -foldxM = P.foldlMx' +foldxM = IsStream.foldlMx' -- | Like 'foldl'' but with a monadic step function. -- @@ -422,7 +446,7 @@ mapM_ f m = S.mapM_ f $ toStreamS m -- @since 0.7.0 {-# INLINE drain #-} drain :: Monad m => SerialT m a -> m () -drain = P.drain +drain = IsStream.drain -- | -- > drainN n = Stream.drain . Stream.take n @@ -513,14 +537,14 @@ headElse x = D.headElse x . toStreamD -- @since 0.1.1 {-# INLINE tail #-} tail :: (IsStream t, Monad m) => SerialT m a -> m (Maybe (t m a)) -tail m = K.tail (K.adapt m) +tail (SerialT m) = fmap (fmap IsStream.fromStream) $ K.tail m -- | Extract all but the last element of the stream, if any. -- -- @since 0.5.0 {-# INLINE init #-} init :: (IsStream t, Monad m) => SerialT m a -> m (Maybe (t m a)) -init m = K.init (K.adapt m) +init (SerialT m) = fmap (fmap IsStream.fromStream) $ K.init m -- | Extract the last element of the stream, if any. -- @@ -657,7 +681,7 @@ minimumBy cmp m = S.minimumBy cmp (toStreamS m) -- @since 0.1.0 {-# INLINE maximum #-} maximum :: (Monad m, Ord a) => SerialT m a -> m (Maybe a) -maximum = P.maximum +maximum m = S.maximum (toStreamS m) -- | Determine the maximum element in a stream using the supplied comparison -- function. @@ -755,7 +779,7 @@ elemIndex a = findIndex (== a) -- @since 0.1.0 {-# INLINE toList #-} toList :: Monad m => SerialT m a -> m [a] -toList = P.toList +toList = IsStream.toList -- | -- @ @@ -789,7 +813,7 @@ toHandle h = go let stop = return () single a = liftIO (IO.hPutStrLn h a) yieldk a r = liftIO (IO.hPutStrLn h a) >> go r - in K.foldStream defState yieldk single stop m1 + in IsStream.foldStream defState yieldk single stop m1 -- | Convert a stream to a pure stream. -- @@ -801,7 +825,7 @@ toHandle h = go -- {-# INLINE toStream #-} toStream :: Monad m => SerialT m a -> m (SerialT n a) -toStream = foldr K.cons K.nil +toStream = foldr IsStream.cons IsStream.nil -- | Convert a stream to a pure stream in reverse order. -- @@ -813,7 +837,7 @@ toStream = foldr K.cons K.nil -- {-# INLINE toStreamRev #-} toStreamRev :: Monad m => SerialT m a -> m (SerialT n a) -toStreamRev = foldl' (flip K.cons) K.nil +toStreamRev = foldl' (flip IsStream.cons) IsStream.nil ------------------------------------------------------------------------------ -- Concurrent Application @@ -853,7 +877,7 @@ toStreamRev = foldl' (flip K.cons) K.nil {-# INLINE (|$.) #-} (|$.) :: (IsStream t, MonadAsync m) => (t m a -> m b) -> (t m a -> m b) -- (|$.) f = f . Async.mkAsync -(|$.) f = f . Par.mkParallel +(|$.) f = f . mkParallel infixr 0 |$. @@ -1009,7 +1033,7 @@ stripSuffix m1 m2 = fmap reverse <$> stripPrefix (reverse m1) (reverse m2) -- @since 0.6.0 {-# INLINABLE eqBy #-} eqBy :: (IsStream t, Monad m) => (a -> b -> Bool) -> t m a -> t m b -> m Bool -eqBy = P.eqBy +eqBy = IsStream.eqBy -- | Compare two streams lexicographically using a comparison function. -- @@ -1018,4 +1042,4 @@ eqBy = P.eqBy cmpBy :: (IsStream t, Monad m) => (a -> b -> Ordering) -> t m a -> t m b -> m Ordering -cmpBy = P.cmpBy +cmpBy = IsStream.cmpBy diff --git a/src/Streamly/Internal/Data/Stream/IsStream/Enumeration.hs b/src/Streamly/Internal/Data/Stream/IsStream/Enumeration.hs index 627fa45774..7594ee545f 100644 --- a/src/Streamly/Internal/Data/Stream/IsStream/Enumeration.hs +++ b/src/Streamly/Internal/Data/Stream/IsStream/Enumeration.hs @@ -60,9 +60,9 @@ import Data.Word import Numeric.Natural import Data.Functor.Identity (Identity(..)) -import Streamly.Internal.Data.Stream.StreamD.Type (fromStreamD) -import Streamly.Internal.Data.Stream.StreamK.Type (IsStream(..)) +import Streamly.Internal.Data.Stream.IsStream.Type (IsStream(..), fromStreamD) +import qualified Streamly.Internal.Data.Stream.IsStream.Type as IsStream import qualified Streamly.Internal.Data.Stream.StreamD.Generate as D import qualified Streamly.Internal.Data.Stream.Serial as Serial (map) @@ -292,8 +292,10 @@ enumerateFromThenToFractional from next to = -- @since 0.6.0 {-# INLINE enumerateFromToSmall #-} enumerateFromToSmall :: (IsStream t, Monad m, Enum a) => a -> a -> t m a -enumerateFromToSmall from to = Serial.map toEnum $ - enumerateFromToIntegral (fromEnum from) (fromEnum to) +enumerateFromToSmall from to = + IsStream.fromSerial + $ Serial.map toEnum + $ enumerateFromToIntegral (fromEnum from) (fromEnum to) -- | 'enumerateFromThenTo' for 'Enum' types not larger than 'Int'. -- @@ -301,8 +303,11 @@ enumerateFromToSmall from to = Serial.map toEnum $ {-# INLINE enumerateFromThenToSmall #-} enumerateFromThenToSmall :: (IsStream t, Monad m, Enum a) => a -> a -> a -> t m a -enumerateFromThenToSmall from next to = Serial.map toEnum $ - enumerateFromThenToIntegral (fromEnum from) (fromEnum next) (fromEnum to) +enumerateFromThenToSmall from next to = + IsStream.fromSerial + $ Serial.map toEnum + $ enumerateFromThenToIntegral + (fromEnum from) (fromEnum next) (fromEnum to) -- | 'enumerateFromThen' for 'Enum' types not larger than 'Int'. -- @@ -550,17 +555,19 @@ ENUMERABLE_FRACTIONAL((Ratio a),Integral a) instance Enumerable a => Enumerable (Identity a) where {-# INLINE enumerateFrom #-} - enumerateFrom (Identity from) = Serial.map Identity $ - enumerateFrom from + enumerateFrom (Identity from) = + IsStream.fromSerial $ Serial.map Identity $ enumerateFrom from {-# INLINE enumerateFromThen #-} - enumerateFromThen (Identity from) (Identity next) = Serial.map Identity $ - enumerateFromThen from next + enumerateFromThen (Identity from) (Identity next) = + IsStream.fromSerial $ Serial.map Identity $ enumerateFromThen from next {-# INLINE enumerateFromTo #-} - enumerateFromTo (Identity from) (Identity to) = Serial.map Identity $ - enumerateFromTo from to + enumerateFromTo (Identity from) (Identity to) = + IsStream.fromSerial $ Serial.map Identity $ enumerateFromTo from to {-# INLINE enumerateFromThenTo #-} enumerateFromThenTo (Identity from) (Identity next) (Identity to) = - Serial.map Identity $ enumerateFromThenTo from next to + IsStream.fromSerial + $ Serial.map Identity + $ enumerateFromThenTo from next to -- TODO {- diff --git a/src/Streamly/Internal/Data/Stream/IsStream/Exception.hs b/src/Streamly/Internal/Data/Stream/IsStream/Exception.hs index 1b1bace772..16da7c6ee1 100644 --- a/src/Streamly/Internal/Data/Stream/IsStream/Exception.hs +++ b/src/Streamly/Internal/Data/Stream/IsStream/Exception.hs @@ -29,8 +29,8 @@ import Control.Monad.IO.Class (MonadIO(..)) import Control.Monad.Trans.Control (MonadBaseControl) import Data.Map.Strict (Map) import Streamly.Internal.Control.Concurrent (MonadAsync) -import Streamly.Internal.Data.Stream.StreamD (toStreamD) -import Streamly.Internal.Data.Stream.StreamK (IsStream) +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream(..), fromStreamD, toStreamD) import qualified Streamly.Internal.Data.Stream.StreamD as D @@ -53,7 +53,7 @@ import qualified Streamly.Internal.Data.Stream.StreamD as D -- @since 0.7.0 {-# INLINE before #-} before :: (IsStream t, Monad m) => m b -> t m a -> t m a -before action xs = D.fromStreamD $ D.before action $ D.toStreamD xs +before action xs = fromStreamD $ D.before action $ toStreamD xs -- | Like 'after', with following differences: -- @@ -70,7 +70,7 @@ before action xs = D.fromStreamD $ D.before action $ D.toStreamD xs -- {-# INLINE after_ #-} after_ :: (IsStream t, Monad m) => m b -> t m a -> t m a -after_ action xs = D.fromStreamD $ D.after_ action $ D.toStreamD xs +after_ action xs = fromStreamD $ D.after_ action $ toStreamD xs -- | Run the action @m b@ whenever the stream @t m a@ stops normally, or if it -- is garbage collected after a partial lazy evaluation. @@ -85,7 +85,7 @@ after_ action xs = D.fromStreamD $ D.after_ action $ D.toStreamD xs {-# INLINE after #-} after :: (IsStream t, MonadIO m, MonadBaseControl IO m) => m b -> t m a -> t m a -after action xs = D.fromStreamD $ D.after action $ D.toStreamD xs +after action xs = fromStreamD $ D.after action $ toStreamD xs -- | Run the action @m b@ if the stream aborts due to an exception. The -- exception is not caught, simply rethrown. @@ -95,7 +95,7 @@ after action xs = D.fromStreamD $ D.after action $ D.toStreamD xs -- @since 0.7.0 {-# INLINE onException #-} onException :: (IsStream t, MonadCatch m) => m b -> t m a -> t m a -onException action xs = D.fromStreamD $ D.onException action $ D.toStreamD xs +onException action xs = fromStreamD $ D.onException action $ toStreamD xs -- | Like 'finally' with following differences: -- @@ -110,7 +110,7 @@ onException action xs = D.fromStreamD $ D.onException action $ D.toStreamD xs -- {-# INLINE finally_ #-} finally_ :: (IsStream t, MonadCatch m) => m b -> t m a -> t m a -finally_ action xs = D.fromStreamD $ D.finally_ action $ D.toStreamD xs +finally_ action xs = fromStreamD $ D.finally_ action $ toStreamD xs -- | Run the action @m b@ whenever the stream @t m a@ stops normally, aborts -- due to an exception or if it is garbage collected after a partial lazy @@ -131,7 +131,7 @@ finally_ action xs = D.fromStreamD $ D.finally_ action $ D.toStreamD xs -- {-# INLINE finally #-} finally :: (IsStream t, MonadAsync m, MonadCatch m) => m b -> t m a -> t m a -finally action xs = D.fromStreamD $ D.finally action $ D.toStreamD xs +finally action xs = fromStreamD $ D.finally action $ toStreamD xs -- | Like 'bracket' but with following differences: -- @@ -148,7 +148,7 @@ finally action xs = D.fromStreamD $ D.finally action $ D.toStreamD xs {-# INLINE bracket_ #-} bracket_ :: (IsStream t, MonadCatch m) => m b -> (b -> m c) -> (b -> t m a) -> t m a -bracket_ bef aft bet = D.fromStreamD $ +bracket_ bef aft bet = fromStreamD $ D.bracket_ bef aft (toStreamD . bet) -- | Run the alloc action @m b@ with async exceptions disabled but keeping @@ -193,7 +193,7 @@ bracket bef aft = bracket' bef aft aft aft {-# INLINE bracket' #-} bracket' :: (IsStream t, MonadAsync m, MonadCatch m) => m b -> (b -> m c) -> (b -> m d) -> (b -> m e) -> (b -> t m a) -> t m a -bracket' bef aft gc exc bet = D.fromStreamD $ +bracket' bef aft gc exc bet = fromStreamD $ D.bracket' bef aft exc gc (toStreamD . bet) -- | Like 'handle' but the exception handler is also provided with the stream @@ -212,9 +212,9 @@ bracket' bef aft gc exc bet = D.fromStreamD $ ghandle :: (IsStream t, MonadCatch m, Exception e) => (e -> t m a -> t m a) -> t m a -> t m a ghandle handler = - D.fromStreamD - . D.ghandle (\e xs -> D.toStreamD $ handler e (D.fromStreamD xs)) - . D.toStreamD + fromStreamD + . D.ghandle (\e xs -> toStreamD $ handler e (fromStreamD xs)) + . toStreamD -- | When evaluating a stream if an exception occurs, stream evaluation aborts -- and the specified exception handler is run with the exception as argument. @@ -226,7 +226,7 @@ ghandle handler = handle :: (IsStream t, MonadCatch m, Exception e) => (e -> t m a) -> t m a -> t m a handle handler xs = - D.fromStreamD $ D.handle (D.toStreamD . handler) $ D.toStreamD xs + fromStreamD $ D.handle (toStreamD . handler) $ toStreamD xs -- | @retry@ takes 3 arguments @@ -269,4 +269,4 @@ retry :: (IsStream t, MonadCatch m, Exception e, Ord e) -> t m a -> t m a retry emap handler inp = - D.fromStreamD $ D.retry emap (D.toStreamD . handler) $ D.toStreamD inp + fromStreamD $ D.retry emap (toStreamD . handler) $ toStreamD inp diff --git a/src/Streamly/Internal/Data/Stream/IsStream/Expand.hs b/src/Streamly/Internal/Data/Stream/IsStream/Expand.hs index b431778310..a6ee74169e 100644 --- a/src/Streamly/Internal/Data/Stream/IsStream/Expand.hs +++ b/src/Streamly/Internal/Data/Stream/IsStream/Expand.hs @@ -30,8 +30,8 @@ module Streamly.Internal.Data.Stream.IsStream.Expand , async , wAsync , parallel - , Par.parallelFst - , Par.parallelMin + , parallelFst + , parallelMin -- * Binary Combinators (Pair Wise) -- | Like the functions in the section above these functions also combine @@ -68,10 +68,10 @@ module Streamly.Internal.Data.Stream.IsStream.Expand , roundrobin -- ** Zip - , Z.zipWith - , Z.zipWithM - , Z.zipAsyncWith - , Z.zipAsyncWithM + , zipWith + , zipWithM + , zipAsyncWith + , zipAsyncWithM -- ** Merge -- , merge @@ -120,9 +120,9 @@ module Streamly.Internal.Data.Stream.IsStream.Expand -- * Flatten Containers -- | Flatten 'Foldable' containers using the binary stream merging -- operations. - , concatFoldableWith - , concatMapFoldableWith - , concatForFoldableWith + , IsStream.concatFoldableWith + , IsStream.concatMapFoldableWith + , IsStream.concatForFoldableWith -- * ConcatMapWith -- | Map and flatten a stream like 'concatMap' but using a custom binary @@ -136,8 +136,8 @@ module Streamly.Internal.Data.Stream.IsStream.Expand -- fashion, a pair wise merging using 'concatPairsWith' would be more -- efficient. These cases include operations like 'mergeBy' or 'zipWith'. - , concatMapWith - , K.bindWith + , IsStream.concatMapWith + , IsStream.bindWith , concatSmapMWith -- * ConcatPairsWith @@ -153,49 +153,56 @@ module Streamly.Internal.Data.Stream.IsStream.Expand -- * Deprecated , concatUnfold + , (<=>) + , (<|) ) where #include "inline.hs" import Streamly.Internal.Control.Concurrent (MonadAsync) -import Streamly.Internal.Data.Stream.Ahead (ahead) -import Streamly.Internal.Data.Stream.Async (async, wAsync) +import Streamly.Internal.Data.Stream.Ahead (aheadK) +import Streamly.Internal.Data.Stream.Async (asyncK, wAsyncK) import Streamly.Internal.Data.Stream.IsStream.Common - (concatM, concatMapM, concatMap, smapM, fromPure, fromEffect) -import Streamly.Internal.Data.Stream.Parallel (parallel) -import Streamly.Internal.Data.Stream.Prelude - ( concatFoldableWith, concatMapFoldableWith - , concatForFoldableWith, fromStreamS, toStreamS) -import Streamly.Internal.Data.Stream.Serial (serial, wSerial) -import Streamly.Internal.Data.Stream.StreamD (fromStreamD, toStreamD) -import Streamly.Internal.Data.Stream.StreamK (IsStream) + ( concatM, concatMapM, concatMap, smapM, fromPure, fromEffect, parallelFst + , zipWith, zipWithM) +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream(..), fromStreamS, toStreamS, fromStreamD, toStreamD) import Streamly.Internal.Data.Unfold.Type (Unfold) +import qualified Streamly.Internal.Data.Stream.IsStream.Type as IsStream import qualified Streamly.Internal.Data.Stream.Parallel as Par import qualified Streamly.Internal.Data.Stream.Serial as Serial import qualified Streamly.Internal.Data.Stream.StreamD as D -import qualified Streamly.Internal.Data.Stream.StreamK as K +import qualified Streamly.Internal.Data.Stream.StreamK.Type as K #ifdef USE_STREAMK_ONLY import qualified Streamly.Internal.Data.Stream.StreamK as S #else import qualified Streamly.Internal.Data.Stream.StreamD as S #endif -import qualified Streamly.Internal.Data.Stream.Zip as Z +import qualified Streamly.Internal.Data.Stream.Zip as Zip -import Prelude hiding (concat, concatMap) +import Prelude hiding (concat, concatMap, zipWith) -- $setup -- >>> :m +-- >>> import Control.Concurrent (threadDelay) -- >>> import Data.IORef -- >>> import Prelude hiding (zipWith, concatMap, concat) -- >>> import qualified Streamly.Prelude as Stream --- >>> import Streamly.Internal.Data.Stream.IsStream as Stream +-- >>> import qualified Streamly.Internal.Data.Stream.IsStream as Stream -- >>> import qualified Streamly.Data.Fold as Fold -- >>> import qualified Streamly.Internal.Data.Fold as Fold -- >>> import qualified Streamly.Internal.Data.Unfold as Unfold -- >>> import qualified Streamly.Internal.Data.Parser as Parser -- >>> import qualified Streamly.Data.Array.Foreign as Array +-- >>> :{ +-- delay n = do +-- threadDelay (n * 1000000) -- sleep for n seconds +-- putStrLn (show n ++ " sec") -- print "n sec" +-- return n -- IO Int +-- :} +-- ------------------------------------------------------------------------------- -- Appending @@ -218,10 +225,79 @@ import Prelude hiding (concat, concatMap) append ::(IsStream t, Monad m) => t m b -> t m b -> t m b append m1 m2 = fromStreamD $ D.append (toStreamD m1) (toStreamD m2) +infixr 6 `serial` + +-- | Appends two streams sequentially, yielding all elements from the first +-- stream, and then all elements from the second stream. +-- +-- >>> import Streamly.Prelude (serial) +-- >>> stream1 = Stream.fromList [1,2] +-- >>> stream2 = Stream.fromList [3,4] +-- >>> Stream.toList $ stream1 `serial` stream2 +-- [1,2,3,4] +-- +-- This operation can be used to fold an infinite lazy container of streams. +-- +-- /Since: 0.2.0 ("Streamly")/ +-- +-- @since 0.8.0 +{-# INLINE serial #-} +serial :: IsStream t => t m a -> t m a -> t m a +serial m1 m2 = fromStream $ K.serial (toStream m1) (toStream m2) + ------------------------------------------------------------------------------- -- Interleaving ------------------------------------------------------------------------------- +infixr 6 `wSerial` + +-- XXX doc duplicated from Stream.Serial module. +-- +-- | Interleaves two streams, yielding one element from each stream +-- alternately. When one stream stops the rest of the other stream is used in +-- the output stream. +-- +-- >>> import Streamly.Prelude (wSerial) +-- >>> stream1 = Stream.fromList [1,2] +-- >>> stream2 = Stream.fromList [3,4] +-- >>> Stream.toList $ Stream.fromWSerial $ stream1 `wSerial` stream2 +-- [1,3,2,4] +-- +-- Note, for singleton streams 'wSerial' and 'serial' are identical. +-- +-- Note that this operation cannot be used to fold a container of infinite +-- streams but it can be used for very large streams as the state that it needs +-- to maintain is proportional to the logarithm of the number of streams. +-- +-- @since 0.8.0 +-- +-- /Since: 0.2.0 ("Streamly")/ + +-- Scheduling Notes: +-- +-- Note that evaluation of @a \`wSerial` b \`wSerial` c@ does not interleave +-- @a@, @b@ and @c@ with equal priority. This expression is equivalent to @a +-- \`wSerial` (b \`wSerial` c)@, therefore, it fairly interleaves @a@ with the +-- result of @b \`wSerial` c@. For example, @Stream.fromList [1,2] \`wSerial` +-- Stream.fromList [3,4] \`wSerial` Stream.fromList [5,6]@ would result in +-- [1,3,2,5,4,6]. In other words, the leftmost stream gets the same scheduling +-- priority as the rest of the streams taken together. The same is true for +-- each subexpression on the right. +-- +{-# INLINE wSerial #-} +wSerial :: IsStream t => t m a -> t m a -> t m a +wSerial m1 m2 = fromStream $ Serial.wSerialK (toStream m1) (toStream m2) + +infixr 5 <=> + +-- | Same as 'wSerial'. +-- +-- @since 0.1.0 +{-# DEPRECATED (<=>) "Please use 'wSerial' instead." #-} +{-# INLINE (<=>) #-} +(<=>) :: IsStream t => t m a -> t m a -> t m a +(<=>) = wSerial + -- XXX Same as 'wSerial'. We should perhaps rename wSerial to interleave. -- XXX Document the interleaving behavior of side effects in all the -- interleaving combinators. @@ -341,6 +417,276 @@ interleaveMin m1 m2 = fromStreamD $ D.interleaveMin (toStreamD m1) (toStreamD m2 roundrobin ::(IsStream t, Monad m) => t m b -> t m b -> t m b roundrobin m1 m2 = fromStreamD $ D.roundRobin (toStreamD m1) (toStreamD m2) +infixr 6 `async` + +-- | Merges two streams, both the streams may be evaluated concurrently, +-- outputs from both are used as they arrive: +-- +-- >>> import Streamly.Prelude (async) +-- >>> stream1 = Stream.fromEffect (delay 4) +-- >>> stream2 = Stream.fromEffect (delay 2) +-- >>> Stream.toList $ stream1 `async` stream2 +-- 2 sec +-- 4 sec +-- [2,4] +-- +-- Multiple streams can be combined. With enough threads, all of them can be +-- scheduled simultaneously: +-- +-- >>> stream3 = Stream.fromEffect (delay 1) +-- >>> Stream.toList $ stream1 `async` stream2 `async` stream3 +-- ... +-- [1,2,4] +-- +-- With 2 threads, only two can be scheduled at a time, when one of those +-- finishes, the third one gets scheduled: +-- +-- >>> Stream.toList $ Stream.maxThreads 2 $ stream1 `async` stream2 `async` stream3 +-- ... +-- [2,1,4] +-- +-- With a single thread, it becomes serial: +-- +-- >>> Stream.toList $ Stream.maxThreads 1 $ stream1 `async` stream2 `async` stream3 +-- ... +-- [4,2,1] +-- +-- Only streams are scheduled for async evaluation, how actions within a +-- stream are evaluated depends on the stream type. If it is a concurrent +-- stream they will be evaluated concurrently. +-- +-- In the following example, both the streams are scheduled for concurrent +-- evaluation but each individual stream is evaluated serially: +-- +-- >>> stream1 = Stream.fromListM $ Prelude.map delay [3,3] -- SerialT IO Int +-- >>> stream2 = Stream.fromListM $ Prelude.map delay [1,1] -- SerialT IO Int +-- >>> Stream.toList $ stream1 `async` stream2 -- IO [Int] +-- ... +-- [1,1,3,3] +-- +-- If total threads are 2, the third stream is scheduled only after one of the +-- first two has finished: +-- +-- > stream3 = Stream.fromListM $ Prelude.map delay [2,2] -- SerialT IO Int +-- > Stream.toList $ Stream.maxThreads 2 $ stream1 `async` stream2 `async` stream3 -- IO [Int] +-- ... +-- [1,1,3,2,3,2] +-- +-- Thus 'async' goes deep in first few streams rather than going wide in all +-- streams. It prefers to evaluate the leftmost streams as much as possible. +-- Because of this behavior, 'async' can be safely used to fold an infinite +-- lazy container of streams. +-- +-- /Since: 0.2.0 ("Streamly")/ +-- +-- @since 0.8.0 +{-# INLINE async #-} +async :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a +async m1 m2 = fromStream $ asyncK (toStream m1) (toStream m2) + +-- | Same as 'async'. +-- +-- @since 0.1.0 +{-# DEPRECATED (<|) "Please use 'async' instead." #-} +{-# INLINE (<|) #-} +(<|) :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a +(<|) = async + +infixr 6 `wAsync` + +-- | For singleton streams, 'wAsync' is the same as 'async'. See 'async' for +-- singleton stream behavior. For multi-element streams, while 'async' is left +-- biased i.e. it tries to evaluate the left side stream as much as possible, +-- 'wAsync' tries to schedule them both fairly. In other words, 'async' goes +-- deep while 'wAsync' goes wide. However, outputs are always used as they +-- arrive. +-- +-- With a single thread, 'async' starts behaving like 'serial' while 'wAsync' +-- starts behaving like 'wSerial'. +-- +-- >>> import Streamly.Prelude (async, wAsync) +-- >>> stream1 = Stream.fromList [1,2,3] +-- >>> stream2 = Stream.fromList [4,5,6] +-- >>> Stream.toList $ Stream.fromAsync $ Stream.maxThreads 1 $ stream1 `async` stream2 +-- [1,2,3,4,5,6] +-- +-- >>> Stream.toList $ Stream.fromWAsync $ Stream.maxThreads 1 $ stream1 `wAsync` stream2 +-- [1,4,2,5,3,6] +-- +-- With two threads available, and combining three streams: +-- +-- >>> stream3 = Stream.fromList [7,8,9] +-- >>> Stream.toList $ Stream.fromAsync $ Stream.maxThreads 2 $ stream1 `async` stream2 `async` stream3 +-- [1,2,3,4,5,6,7,8,9] +-- +-- >>> Stream.toList $ Stream.fromWAsync $ Stream.maxThreads 2 $ stream1 `wAsync` stream2 `wAsync` stream3 +-- [1,4,2,7,5,3,8,6,9] +-- +-- This operation cannot be used to fold an infinite lazy container of streams, +-- because it schedules all the streams in a round robin manner. +-- +-- Note that 'WSerialT' and single threaded 'WAsyncT' both interleave streams +-- but the exact scheduling is slightly different in both cases. +-- +-- @since 0.8.0 +-- +-- /Since: 0.2.0 ("Streamly")/ + +-- Scheduling details: +-- +-- This is how the execution of the above example proceeds: +-- +-- 1. The scheduler queue is initialized with @[S.fromList [1,2,3], +-- (S.fromList [4,5,6]) \<> (S.fromList [7,8,9])]@ assuming the head of the +-- queue is represented by the rightmost item. +-- 2. @S.fromList [1,2,3]@ is executed, yielding the element @1@ and putting +-- @[2,3]@ at the back of the scheduler queue. The scheduler queue now looks +-- like @[(S.fromList [4,5,6]) \<> (S.fromList [7,8,9]), S.fromList [2,3]]@. +-- 3. Now @(S.fromList [4,5,6]) \<> (S.fromList [7,8,9])@ is picked up for +-- execution, @S.fromList [7,8,9]@ is added at the back of the queue and +-- @S.fromList [4,5,6]@ is executed, yielding the element @4@ and adding +-- @S.fromList [5,6]@ at the back of the queue. The queue now looks like +-- @[S.fromList [2,3], S.fromList [7,8,9], S.fromList [5,6]]@. +-- 4. Note that the scheduler queue expands by one more stream component in +-- every pass because one more @<>@ is broken down into two components. At this +-- point there are no more @<>@ operations to be broken down further and the +-- queue has reached its maximum size. Now these streams are scheduled in +-- round-robin fashion yielding @[2,7,5,3,8,6,9]@. +-- +-- As we see above, in a right associated expression composed with @<>@, only +-- one @<>@ operation is broken down into two components in one execution, +-- therefore, if we have @n@ streams composed using @<>@ it will take @n@ +-- scheduler passes to expand the whole expression. By the time @n-th@ +-- component is added to the scheduler queue, the first component would have +-- received @n@ scheduler passes. +-- +{-# INLINE wAsync #-} +wAsync :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a +wAsync m1 m2 = fromStream $ wAsyncK (toStream m1) (toStream m2) + +infixr 6 `ahead` + +-- | Appends two streams, both the streams may be evaluated concurrently but +-- the outputs are used in the same order as the corresponding actions in the +-- original streams, side effects will happen in the order in which the streams +-- are evaluated: +-- +-- >>> import Streamly.Prelude (ahead, SerialT) +-- >>> stream1 = Stream.fromEffect (delay 4) :: SerialT IO Int +-- >>> stream2 = Stream.fromEffect (delay 2) :: SerialT IO Int +-- >>> Stream.toList $ stream1 `ahead` stream2 :: IO [Int] +-- 2 sec +-- 4 sec +-- [4,2] +-- +-- Multiple streams can be combined. With enough threads, all of them can be +-- scheduled simultaneously: +-- +-- >>> stream3 = Stream.fromEffect (delay 1) +-- >>> Stream.toList $ stream1 `ahead` stream2 `ahead` stream3 +-- 1 sec +-- 2 sec +-- 4 sec +-- [4,2,1] +-- +-- With 2 threads, only two can be scheduled at a time, when one of those +-- finishes, the third one gets scheduled: +-- +-- >>> Stream.toList $ Stream.maxThreads 2 $ stream1 `ahead` stream2 `ahead` stream3 +-- 2 sec +-- 1 sec +-- 4 sec +-- [4,2,1] +-- +-- Only streams are scheduled for ahead evaluation, how actions within a stream +-- are evaluated depends on the stream type. If it is a concurrent stream they +-- will be evaluated concurrently. It may not make much sense combining serial +-- streams using 'ahead'. +-- +-- 'ahead' can be safely used to fold an infinite lazy container of streams. +-- +-- /Since: 0.3.0 ("Streamly")/ +-- +-- @since 0.8.0 +{-# INLINE ahead #-} +ahead :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a +ahead m1 m2 = fromStream $ aheadK (toStream m1) (toStream m2) + +infixr 6 `parallel` + +-- | Like 'Streamly.Prelude.async' except that the execution is much more +-- strict. There is no limit on the number of threads. While +-- 'Streamly.Prelude.async' may not schedule a stream if there is no demand +-- from the consumer, 'parallel' always evaluates both the streams immediately. +-- The only limit that applies to 'parallel' is 'Streamly.Prelude.maxBuffer'. +-- Evaluation may block if the output buffer becomes full. +-- +-- >>> import Streamly.Prelude (parallel) +-- >>> stream = Stream.fromEffect (delay 2) `parallel` Stream.fromEffect (delay 1) +-- >>> Stream.toList stream -- IO [Int] +-- 1 sec +-- 2 sec +-- [1,2] +-- +-- 'parallel' guarantees that all the streams are scheduled for execution +-- immediately, therefore, we could use things like starting timers inside the +-- streams and relying on the fact that all timers were started at the same +-- time. +-- +-- Unlike 'async' this operation cannot be used to fold an infinite lazy +-- container of streams, because it schedules all the streams strictly +-- concurrently. +-- +-- /Since: 0.2.0 ("Streamly")/ +-- +-- @since 0.8.0 +{-# INLINE parallel #-} +parallel :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a +parallel m1 m2 = fromStream $ Par.parallelK (toStream m1) (toStream m2) + +-- This is a race like combinator for streams. +-- +-- | Like `parallel` but stops the output as soon as any of the two streams +-- stops. +-- +-- /Pre-release/ +{-# INLINE parallelMin #-} +parallelMin :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a +parallelMin m1 m2 = fromStream $ Par.parallelMinK (toStream m1) (toStream m2) + +------------------------------------------------------------------------------ +-- Zipping +------------------------------------------------------------------------------ + +-- | Like 'zipAsyncWith' but with a monadic zipping function. +-- +-- @since 0.4.0 +{-# INLINE zipAsyncWithM #-} +zipAsyncWithM :: (IsStream t, MonadAsync m) + => (a -> b -> m c) -> t m a -> t m b -> t m c +zipAsyncWithM f m1 m2 = + fromStream $ Zip.zipAsyncWithMK f (toStream m1) (toStream m2) + +-- XXX Should we rename this to zipParWith or zipParallelWith? This can happen +-- along with the change of behvaior to end the stream concurrently. +-- +-- | Like 'zipWith' but zips concurrently i.e. both the streams being zipped +-- are evaluated concurrently using the 'ParallelT' concurrent evaluation +-- style. The maximum number of elements of each stream evaluated in advance +-- can be controlled by 'maxBuffer'. +-- +-- The stream ends if stream @a@ or stream @b@ ends. However, if stream @b@ +-- ends while we are still evaluating stream @a@ and waiting for a result then +-- stream will not end until after the evaluation of stream @a@ finishes. This +-- behavior can potentially be changed in future to end the stream immediately +-- as soon as any of the stream end is detected. +-- +-- @since 0.1.0 +{-# INLINE zipAsyncWith #-} +zipAsyncWith :: (IsStream t, MonadAsync m) + => (a -> b -> c) -> t m a -> t m b -> t m c +zipAsyncWith f = zipAsyncWithM (\a b -> return (f a b)) + ------------------------------------------------------------------------------ -- Merging (sorted streams) ------------------------------------------------------------------------------ @@ -510,7 +856,7 @@ unfoldManyRoundRobin u m = interpose :: (IsStream t, Monad m) => c -> Unfold m b c -> t m b -> t m c interpose x unf str = - D.fromStreamD $ D.interpose (return x) unf (D.toStreamD str) + fromStreamD $ D.interpose (return x) unf (toStreamD str) -- interposeSuffix x unf str = gintercalateSuffix unf str UF.identity (repeat x) -- @@ -524,7 +870,7 @@ interpose x unf str = interposeSuffix :: (IsStream t, Monad m) => c -> Unfold m b c -> t m b -> t m c interposeSuffix x unf str = - D.fromStreamD $ D.interposeSuffix (return x) unf (D.toStreamD str) + fromStreamD $ D.interposeSuffix (return x) unf (toStreamD str) ------------------------------------------------------------------------------ -- Combine N Streams - intercalate @@ -546,9 +892,9 @@ gintercalate :: (IsStream t, Monad m) => Unfold m a c -> t m a -> Unfold m b c -> t m b -> t m c gintercalate unf1 str1 unf2 str2 = - D.fromStreamD $ D.gintercalate - unf1 (D.toStreamD str1) - unf2 (D.toStreamD str2) + fromStreamD $ D.gintercalate + unf1 (toStreamD str1) + unf2 (toStreamD str2) -- > intercalate unf seed str = gintercalate unf str unf (repeatM seed) -- @@ -565,7 +911,7 @@ gintercalate unf1 str1 unf2 str2 = {-# INLINE intercalate #-} intercalate :: (IsStream t, Monad m) => Unfold m b c -> b -> t m b -> t m c -intercalate unf seed str = D.fromStreamD $ +intercalate unf seed str = fromStreamD $ D.unfoldMany unf $ D.intersperse seed (toStreamD str) -- | 'interleaveSuffix' followed by unfold and concat. @@ -576,9 +922,9 @@ gintercalateSuffix :: (IsStream t, Monad m) => Unfold m a c -> t m a -> Unfold m b c -> t m b -> t m c gintercalateSuffix unf1 str1 unf2 str2 = - D.fromStreamD $ D.gintercalateSuffix - unf1 (D.toStreamD str1) - unf2 (D.toStreamD str2) + fromStreamD $ D.gintercalateSuffix + unf1 (toStreamD str1) + unf2 (toStreamD str2) -- > intercalateSuffix unf seed str = gintercalateSuffix unf str unf (repeatM seed) -- @@ -596,7 +942,7 @@ gintercalateSuffix unf1 str1 unf2 str2 = intercalateSuffix :: (IsStream t, Monad m) => Unfold m b c -> b -> t m b -> t m c intercalateSuffix unf seed str = fromStreamD $ D.unfoldMany unf - $ D.intersperseSuffix (return seed) (D.toStreamD str) + $ D.intersperseSuffix (return seed) (toStreamD str) {- {-# INLINE iterateUnfold #-} @@ -624,25 +970,6 @@ concat = concatMap id -- Combine N Streams - concatMap ------------------------------------------------------------------------------ --- | @concatMapWith mixer generator stream@ is a two dimensional looping --- combinator. The @generator@ function is used to generate streams from the --- elements in the input @stream@ and the @mixer@ function is used to merge --- those streams. --- --- Note we can merge streams concurrently by using a concurrent merge function. --- --- /Since: 0.7.0/ --- --- /Since: 0.8.0 (signature change)/ -{-# INLINE concatMapWith #-} -concatMapWith - :: IsStream t - => (t m b -> t m b -> t m b) - -> (a -> t m b) - -> t m a - -> t m b -concatMapWith = K.concatMapBy - -- | Like 'concatMapWith' but carries a state which can be used to share -- information across multiple steps of concat. -- @@ -660,7 +987,8 @@ concatSmapMWith -> m s -> t m a -> t m b -concatSmapMWith combine f initial = concatMapWith combine id . smapM f initial +concatSmapMWith combine f initial = + IsStream.concatMapWith combine id . smapM f initial -- Keep concating either streams as long as rights are generated, stop as soon -- as a left is generated and concat the left stream. @@ -700,7 +1028,12 @@ concatPairsWith :: IsStream t => -> (a -> t m b) -> t m a -> t m b -concatPairsWith = K.concatPairsWith +concatPairsWith par f m = + fromStream + $ K.concatPairsWith + (\s1 s2 -> toStream $ fromStream s1 `par` fromStream s2) + (toStream . f) + (toStream m) ------------------------------------------------------------------------------ -- IterateMap - Map and flatten Trees of Streams @@ -737,9 +1070,9 @@ iterateMapWith -> (a -> t m a) -> t m a -> t m a -iterateMapWith combine f = concatMapWith combine go +iterateMapWith combine f = IsStream.concatMapWith combine go where - go x = fromPure x `combine` concatMapWith combine go (f x) + go x = fromPure x `combine` IsStream.concatMapWith combine go (f x) {- {-# INLINE iterateUnfold #-} @@ -776,7 +1109,9 @@ iterateSmapMWith -> t m a -> t m a iterateSmapMWith combine f initial stream = - concatMap (\b -> concatMapWith combine (go b) stream) (fromEffect initial) + concatMap + (\b -> IsStream.concatMapWith combine (go b) stream) + (fromEffect initial) where @@ -784,7 +1119,7 @@ iterateSmapMWith combine f initial stream = feedback b a = concatMap - (\(b1, s) -> concatMapWith combine (go b1) s) + (\(b1, s) -> IsStream.concatMapWith combine (go b1) s) (fromEffect $ f b a) ------------------------------------------------------------------------------ @@ -813,4 +1148,5 @@ iterateMapLeftsWith -> (a -> t m (Either a b)) -> t m (Either a b) -> t m (Either a b) -iterateMapLeftsWith combine f = iterateMapWith combine (either f (const K.nil)) +iterateMapLeftsWith combine f = + iterateMapWith combine (either f (const IsStream.nil)) diff --git a/src/Streamly/Internal/Data/Stream/IsStream/Generate.hs b/src/Streamly/Internal/Data/Stream/IsStream/Generate.hs index 4dc45b5e08..fe78456cd0 100644 --- a/src/Streamly/Internal/Data/Stream/IsStream/Generate.hs +++ b/src/Streamly/Internal/Data/Stream/IsStream/Generate.hs @@ -18,10 +18,10 @@ module Streamly.Internal.Data.Stream.IsStream.Generate ( -- * Primitives - K.nil - , K.nilM - , K.cons - , (K..:) + IsStream.nil + , IsStream.nilM + , IsStream.cons + , (IsStream..:) , consM , (|:) @@ -68,17 +68,18 @@ module Streamly.Internal.Data.Stream.IsStream.Generate , iterateM -- * Cyclic Elements - , K.mfix + , mfix -- * From Containers - , P.fromList + , IsStream.fromList , fromListM - , K.fromFoldable + , fromFoldable , fromFoldableM , fromCallback + , fromPrimIORef -- * Deprecated - , K.once + , once , yield , yieldM , each @@ -90,6 +91,7 @@ where #include "inline.hs" import Control.Monad.IO.Class (MonadIO(..)) +import Data.Primitive.Types (Prim) import Data.Void (Void) import Streamly.Internal.Control.Concurrent (MonadAsync) import Streamly.Internal.Data.Unfold.Type (Unfold) @@ -99,20 +101,21 @@ import Streamly.Internal.Data.Stream.IsStream.Enumeration import Streamly.Internal.Data.Stream.IsStream.Common ( absTimesWith, concatM, relTimesWith, timesWith, fromPure, fromEffect , yield, yieldM, repeatM) -import Streamly.Internal.Data.Stream.Prelude (fromStreamS) -import Streamly.Internal.Data.Stream.StreamD (fromStreamD) -import Streamly.Internal.Data.Stream.StreamK (IsStream((|:), consM)) -import Streamly.Internal.Data.Stream.Serial (SerialT, WSerialT) +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream (..), fromSerial, consM, fromStreamD, fromStreamS) +import Streamly.Internal.Data.Stream.Serial (SerialT(..), WSerialT) import Streamly.Internal.Data.Stream.Zip (ZipSerialM) import Streamly.Internal.Data.Time.Units (AbsTime , RelTime64, addToAbsTime64) +import qualified Streamly.Internal.Data.IORef.Prim as Prim +import qualified Streamly.Internal.Data.Stream.IsStream.Type as IsStream import qualified Streamly.Internal.Data.Stream.Parallel as Par -import qualified Streamly.Internal.Data.Stream.Prelude as P import qualified Streamly.Internal.Data.Stream.Serial as Serial import qualified Streamly.Internal.Data.Stream.StreamD.Generate as D -import qualified Streamly.Internal.Data.Stream.StreamK as K +import qualified Streamly.Internal.Data.Stream.StreamK.Type as K #ifdef USE_STREAMK_ONLY import qualified Streamly.Internal.Data.Stream.StreamK as S +import qualified Streamly.Internal.Data.Stream.StreamK.Type as S #else import qualified Streamly.Internal.Data.Stream.StreamD.Generate as S #endif @@ -217,8 +220,9 @@ unfoldr step seed = fromStreamS (S.unfoldr step seed) -- -- /Since: 0.1.0/ {-# INLINE_EARLY unfoldrM #-} -unfoldrM :: (IsStream t, MonadAsync m) => (b -> m (Maybe (a, b))) -> b -> t m a -unfoldrM = K.unfoldrM +unfoldrM :: forall t m a b. (IsStream t, MonadAsync m) => + (b -> m (Maybe (a, b))) -> b -> t m a +unfoldrM step = fromStream . K.unfoldrMWith (IsStream.toConsK (consM @t)) step {-# RULES "unfoldrM serial" unfoldrM = unfoldrMSerial #-} {-# INLINE_EARLY unfoldrMSerial #-} @@ -228,12 +232,13 @@ unfoldrMSerial = Serial.unfoldrM {-# RULES "unfoldrM wSerial" unfoldrM = unfoldrMWSerial #-} {-# INLINE_EARLY unfoldrMWSerial #-} unfoldrMWSerial :: MonadAsync m => (b -> m (Maybe (a, b))) -> b -> WSerialT m a -unfoldrMWSerial = Serial.unfoldrM +unfoldrMWSerial f = fromSerial . Serial.unfoldrM f {-# RULES "unfoldrM zipSerial" unfoldrM = unfoldrMZipSerial #-} {-# INLINE_EARLY unfoldrMZipSerial #-} -unfoldrMZipSerial :: MonadAsync m => (b -> m (Maybe (a, b))) -> b -> ZipSerialM m a -unfoldrMZipSerial = Serial.unfoldrM +unfoldrMZipSerial :: MonadAsync m => + (b -> m (Maybe (a, b))) -> b -> ZipSerialM m a +unfoldrMZipSerial f = fromSerial . Serial.unfoldrM f ------------------------------------------------------------------------------ -- From Values @@ -275,8 +280,9 @@ replicate n = fromStreamS . S.replicate n -- -- @since 0.1.1 {-# INLINE_EARLY replicateM #-} -replicateM :: (IsStream t, MonadAsync m) => Int -> m a -> t m a -replicateM = K.replicateM +replicateM :: forall t m a. (IsStream t, MonadAsync m) => Int -> m a -> t m a +replicateM count = + fromStream . K.replicateMWith (IsStream.toConsK (consM @t)) count {-# RULES "replicateM serial" replicateM = replicateMSerial #-} {-# INLINE replicateMSerial #-} @@ -422,8 +428,9 @@ fromIndices = fromStreamS . S.fromIndices -- -- @since 0.6.0 {-# INLINE_EARLY fromIndicesM #-} -fromIndicesM :: (IsStream t, MonadAsync m) => (Int -> m a) -> t m a -fromIndicesM = K.fromIndicesM +fromIndicesM :: forall t m a. (IsStream t, MonadAsync m) => + (Int -> m a) -> t m a +fromIndicesM = fromStream . K.fromIndicesMWith (IsStream.toConsK (consM @t)) {-# RULES "fromIndicesM serial" fromIndicesM = fromIndicesMSerial #-} {-# INLINE fromIndicesMSerial #-} @@ -482,18 +489,90 @@ iterate step = fromStreamS . S.iterate step -- -- /Since: 0.7.0 (signature change)/ {-# INLINE_EARLY iterateM #-} -iterateM :: (IsStream t, MonadAsync m) => (a -> m a) -> m a -> t m a -iterateM = K.iterateM +iterateM :: forall t m a. (IsStream t, MonadAsync m) => + (a -> m a) -> m a -> t m a +iterateM f = fromStream . K.iterateMWith (IsStream.toConsK (consM @t)) f {-# RULES "iterateM serial" iterateM = iterateMSerial #-} {-# INLINE iterateMSerial #-} iterateMSerial :: MonadAsync m => (a -> m a) -> m a -> SerialT m a iterateMSerial step = fromStreamS . S.iterateM step +-- | We can define cyclic structures using @let@: +-- +-- >>> let (a, b) = ([1, b], head a) in (a, b) +-- ([1,1],1) +-- +-- The function @fix@ defined as: +-- +-- > fix f = let x = f x in x +-- +-- ensures that the argument of a function and its output refer to the same +-- lazy value @x@ i.e. the same location in memory. Thus @x@ can be defined +-- in terms of itself, creating structures with cyclic references. +-- +-- >>> import Data.Function (fix) +-- >>> f ~(a, b) = ([1, b], head a) +-- >>> fix f +-- ([1,1],1) +-- +-- 'Control.Monad.mfix' is essentially the same as @fix@ but for monadic +-- values. +-- +-- Using 'mfix' for streams we can construct a stream in which each element of +-- the stream is defined in a cyclic fashion. The argument of the function +-- being fixed represents the current element of the stream which is being +-- returned by the stream monad. Thus, we can use the argument to construct +-- itself. +-- +-- In the following example, the argument @action@ of the function @f@ +-- represents the tuple @(x,y)@ returned by it in a given iteration. We define +-- the first element of the tuple in terms of the second. +-- +-- @ +-- import Streamly.Internal.Data.Stream.IsStream as Stream +-- import System.IO.Unsafe (unsafeInterleaveIO) +-- +-- main = do +-- Stream.mapM_ print $ Stream.mfix f +-- +-- where +-- +-- f action = do +-- let incr n act = fmap ((+n) . snd) $ unsafeInterleaveIO act +-- x <- Stream.fromListM [incr 1 action, incr 2 action] +-- y <- Stream.fromList [4,5] +-- return (x, y) +-- @ +-- +-- Note: you cannot achieve this by just changing the order of the monad +-- statements because that would change the order in which the stream elements +-- are generated. +-- +-- Note that the function @f@ must be lazy in its argument, that's why we use +-- 'unsafeInterleaveIO' on @action@ because IO monad is strict. +-- +-- /Pre-release/ +{-# INLINE mfix #-} +mfix :: (IsStream t, Monad m) => (m a -> t m a) -> t m a +mfix f = fromStream $ K.mfix (toStream . f) + ------------------------------------------------------------------------------ -- Conversions ------------------------------------------------------------------------------ +-- | +-- @ +-- fromFoldable = 'Prelude.foldr' 'cons' 'nil' +-- @ +-- +-- Construct a stream from a 'Foldable' containing pure values: +-- +-- @since 0.2.0 +{-# INLINE fromFoldable #-} +fromFoldable :: (IsStream t, Foldable f) => f a -> t m a +fromFoldable = fromStream . K.fromFoldable + -- | -- @ -- fromFoldableM = 'Prelude.foldr' 'consM' 'K.nil' @@ -511,7 +590,7 @@ iterateMSerial step = fromStreamS . S.iterateM step -- @since 0.3.0 {-# INLINE fromFoldableM #-} fromFoldableM :: (IsStream t, MonadAsync m, Foldable f) => f (m a) -> t m a -fromFoldableM = Prelude.foldr consM K.nil +fromFoldableM = Prelude.foldr consM IsStream.nil -- | -- @ @@ -526,7 +605,7 @@ fromFoldableM = Prelude.foldr consM K.nil fromListM :: (MonadAsync m, IsStream t) => [m a] -> t m a fromListM = fromFoldableM {-# RULES "fromListM fallback to StreamK" [1] - forall a. D.toStreamK (D.fromListM a) = fromFoldableM a #-} + forall a. D.toStreamK (D.fromListM a) = K.fromFoldableM a #-} {-# RULES "fromListM serial" fromListM = fromListMSerial #-} {-# INLINE_EARLY fromListMSerial #-} @@ -539,7 +618,7 @@ fromListMSerial = fromStreamD . D.fromListM {-# DEPRECATED each "Please use fromFoldable instead." #-} {-# INLINE each #-} each :: (IsStream t, Foldable f) => f a -> t m a -each = K.fromFoldable +each = fromFoldable -- | Read lines from an IO Handle into a stream of Strings. -- @@ -549,7 +628,7 @@ each = K.fromFoldable fromHandle :: (IsStream t, MonadIO m) => IO.Handle -> t m String fromHandle h = go where - go = K.mkStream $ \_ yld _ stp -> do + go = IsStream.mkStream $ \_ yld _ stp -> do eof <- liftIO $ IO.hIsEOF h if eof then stp @@ -570,4 +649,20 @@ fromCallback :: MonadAsync m => ((a -> m ()) -> m ()) -> SerialT m a fromCallback setCallback = concatM $ do (callback, stream) <- Par.newCallbackStream setCallback callback - return stream + return $ SerialT stream + +-- | Construct a stream by reading a 'Prim' 'IORef' repeatedly. +-- +-- /Pre-release/ +-- +{-# INLINE fromPrimIORef #-} +fromPrimIORef :: (IsStream t, MonadIO m, Prim a) => Prim.IORef a -> t m a +fromPrimIORef = fromStreamD . Prim.toStreamD + +-- | Same as fromEffect +-- +-- @since 0.2.0 +{-# DEPRECATED once "Please use fromEffect instead." #-} +{-# INLINE once #-} +once :: (Monad m, IsStream t) => m a -> t m a +once = IsStream.fromEffect diff --git a/src/Streamly/Internal/Data/Stream/IsStream/Lift.hs b/src/Streamly/Internal/Data/Stream/IsStream/Lift.hs index 8540bc6a04..71cb0bfe48 100644 --- a/src/Streamly/Internal/Data/Stream/IsStream/Lift.hs +++ b/src/Streamly/Internal/Data/Stream/IsStream/Lift.hs @@ -28,10 +28,9 @@ import Control.Monad.Trans.Reader (ReaderT) import Control.Monad.Trans.State.Strict (StateT) import Control.Monad.Trans.Class (MonadTrans(..)) import Data.Functor.Identity (Identity (..)) -import Streamly.Internal.Data.Stream.Prelude (fromStreamS, toStreamS) +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream(..), fromStreamS, toStreamS, fromStreamD, toStreamD) import Streamly.Internal.Data.Stream.Serial (SerialT) -import Streamly.Internal.Data.Stream.StreamD (fromStreamD, toStreamD) -import Streamly.Internal.Data.Stream.StreamK (IsStream) import qualified Streamly.Internal.Data.Stream.StreamD as D #ifdef USE_STREAMK_ONLY diff --git a/src/Streamly/Internal/Data/Stream/IsStream/Reduce.hs b/src/Streamly/Internal/Data/Stream/IsStream/Reduce.hs index a39727a4af..76229279ac 100644 --- a/src/Streamly/Internal/Data/Stream/IsStream/Reduce.hs +++ b/src/Streamly/Internal/Data/Stream/IsStream/Reduce.hs @@ -163,11 +163,14 @@ import Streamly.Internal.Data.Stream.IsStream.Common , fold , interjectSuffix , intersperseM + , map + , parallelFst , repeatM , scanlMAfter' , splitOnSeq , fromPure) -import Streamly.Internal.Data.Stream.StreamK (IsStream) +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream(..), fromStreamD, toStreamD, cons) import Streamly.Internal.Data.Time.Units ( AbsTime, MilliSecond64(..), addToAbsTime, toRelTime , toAbsTime) @@ -179,12 +182,10 @@ import qualified Streamly.Internal.Data.Array.Foreign.Type as A import qualified Streamly.Internal.Data.Fold as FL import qualified Streamly.Internal.Data.Parser.ParserK.Type as PRK import qualified Streamly.Internal.Data.Parser.ParserD as PRD -import qualified Streamly.Internal.Data.Stream.Parallel as Par -import qualified Streamly.Internal.Data.Stream.Serial as Serial +import qualified Streamly.Internal.Data.Stream.IsStream.Type as IsStream import qualified Streamly.Internal.Data.Stream.StreamD as D -import qualified Streamly.Internal.Data.Stream.StreamK as K -import Prelude hiding (concatMap) +import Prelude hiding (concatMap, map) -- $setup -- >>> :m @@ -265,7 +266,7 @@ foldManyPost => Fold m a b -> t m a -> t m b -foldManyPost f m = D.fromStreamD $ D.foldManyPost f (D.toStreamD m) +foldManyPost f m = fromStreamD $ D.foldManyPost f (toStreamD m) -- | Apply a 'Fold' repeatedly on a stream and emit the fold outputs in the -- output stream. @@ -292,7 +293,7 @@ foldMany => Fold m a b -> t m a -> t m b -foldMany f m = D.fromStreamD $ D.foldMany f (D.toStreamD m) +foldMany f m = fromStreamD $ D.foldMany f (toStreamD m) -- | Apply a stream of folds to an input stream and emit the results in the -- output stream. @@ -328,7 +329,7 @@ foldSequence _f _m = undefined {-# INLINE foldIterateM #-} foldIterateM :: (IsStream t, Monad m) => (b -> m (Fold m a b)) -> b -> t m a -> t m b -foldIterateM f i m = D.fromStreamD $ D.foldIterateM f i (D.toStreamD m) +foldIterateM f i m = fromStreamD $ D.foldIterateM f i (toStreamD m) ------------------------------------------------------------------------------ -- Parsing @@ -365,7 +366,7 @@ parseMany -> t m a -> t m b parseMany p m = - D.fromStreamD $ D.parseMany (PRK.fromParserK p) (D.toStreamD m) + fromStreamD $ D.parseMany (PRK.fromParserK p) (toStreamD m) {-# INLINE parseManyD #-} parseManyD @@ -374,7 +375,7 @@ parseManyD -> t m a -> t m b parseManyD p m = - D.fromStreamD $ D.parseMany p (D.toStreamD m) + fromStreamD $ D.parseMany p (toStreamD m) -- | Apply a stream of parsers to an input stream and emit the results in the -- output stream. @@ -428,8 +429,8 @@ parseIterate -> b -> t m a -> t m b -parseIterate f i m = D.fromStreamD $ - D.parseIterate (PRK.fromParserK . f) i (D.toStreamD m) +parseIterate f i m = fromStreamD $ + D.parseIterate (PRK.fromParserK . f) i (toStreamD m) ------------------------------------------------------------------------------ -- Generalized grouping @@ -492,7 +493,7 @@ groupsBy -> Fold m a b -> t m a -> t m b -groupsBy cmp f m = D.fromStreamD $ D.groupsBy cmp f (D.toStreamD m) +groupsBy cmp f m = fromStreamD $ D.groupsBy cmp f (toStreamD m) -- | Unlike @groupsBy@ this function performs a rolling comparison of two -- successive elements in the input stream. @groupsByRolling cmp f $ S.fromList @@ -513,7 +514,7 @@ groupsByRolling -> Fold m a b -> t m a -> t m b -groupsByRolling cmp f m = D.fromStreamD $ D.groupsRollingBy cmp f (D.toStreamD m) +groupsByRolling cmp f m = fromStreamD $ D.groupsRollingBy cmp f (toStreamD m) -- | -- > groups = groupsBy (==) @@ -736,7 +737,7 @@ wordsBy :: (IsStream t, Monad m) => (a -> Bool) -> Fold m a b -> t m a -> t m b wordsBy predicate f m = - D.fromStreamD $ D.wordsBy predicate f (D.toStreamD m) + fromStreamD $ D.wordsBy predicate f (toStreamD m) -- | Like 'splitOnSuffix' but keeps the suffix attached to the resulting -- splits. @@ -812,7 +813,8 @@ splitWithSuffix predicate f = foldMany (FL.takeEndBy predicate f) splitOnAny :: (IsStream t, Monad m, Storable a, Integral a) => [Array a] -> Fold m a b -> t m a -> t m b -splitOnAny subseq f m = undefined -- D.fromStreamD $ D.splitOnAny f subseq (D.toStreamD m) +splitOnAny subseq f m = undefined + -- fromStreamD $ D.splitOnAny f subseq (toStreamD m) -} -- XXX use a non-monadic intersperse to remove the MonadAsync constraint. @@ -907,7 +909,7 @@ splitOnSuffixSeq :: (IsStream t, MonadIO m, Storable a, Enum a, Eq a) => Array a -> Fold m a b -> t m a -> t m b splitOnSuffixSeq patt f m = - D.fromStreamD $ D.splitOnSuffixSeq False patt f (D.toStreamD m) + fromStreamD $ D.splitOnSuffixSeq False patt f (toStreamD m) {- -- | Like 'splitOn' but drops any empty splits. @@ -916,7 +918,8 @@ splitOnSuffixSeq patt f m = wordsOn :: (IsStream t, Monad m, Storable a, Eq a) => Array a -> Fold m a b -> t m a -> t m b -wordsOn subseq f m = undefined -- D.fromStreamD $ D.wordsOn f subseq (D.toStreamD m) +wordsOn subseq f m = undefined + -- fromStreamD $ D.wordsOn f subseq (toStreamD m) -} -- | Like 'splitOnSuffixSeq' but keeps the suffix intact in the splits. @@ -953,7 +956,7 @@ splitWithSuffixSeq :: (IsStream t, MonadIO m, Storable a, Enum a, Eq a) => Array a -> Fold m a b -> t m a -> t m b splitWithSuffixSeq patt f m = - D.fromStreamD $ D.splitOnSuffixSeq True patt f (D.toStreamD m) + fromStreamD $ D.splitOnSuffixSeq True patt f (toStreamD m) {- -- This can be implemented easily using Rabin Karp @@ -963,7 +966,7 @@ splitOnSuffixSeqAny :: (IsStream t, Monad m, Storable a, Integral a) => [Array a] -> Fold m a b -> t m a -> t m b splitOnSuffixSeqAny subseq f m = undefined - -- D.fromStreamD $ D.splitPostAny f subseq (D.toStreamD m) + -- fromStreamD $ D.splitPostAny f subseq (toStreamD m) -} ------------------------------------------------------------------------------ @@ -986,7 +989,7 @@ splitOnSuffixSeqAny subseq f m = undefined chunksOf :: (IsStream t, Monad m) => Int -> Fold m a b -> t m a -> t m b -chunksOf n f = D.fromStreamD . D.chunksOf n f . D.toStreamD +chunksOf n f = fromStreamD . D.chunksOf n f . toStreamD -- | -- @@ -995,7 +998,7 @@ chunksOf n f = D.fromStreamD . D.chunksOf n f . D.toStreamD chunksOf2 :: (IsStream t, Monad m) => Int -> m c -> Fold2 m c a b -> t m a -> t m b -chunksOf2 n action f m = D.fromStreamD $ D.groupsOf2 n action f (D.toStreamD m) +chunksOf2 n action f m = fromStreamD $ D.groupsOf2 n action f (toStreamD m) -- | @arraysOf n stream@ groups the elements in the input stream into arrays of -- @n@ elements each. @@ -1008,7 +1011,7 @@ chunksOf2 n action f m = D.fromStreamD $ D.groupsOf2 n action f (D.toStreamD m) {-# INLINE arraysOf #-} arraysOf :: (IsStream t, MonadIO m, Storable a) => Int -> t m a -> t m (Array a) -arraysOf n = D.fromStreamD . A.arraysOf n . D.toStreamD +arraysOf n = fromStreamD . A.arraysOf n . toStreamD -- XXX we can implement this by repeatedly applying the 'lrunFor' fold. -- XXX add this example after fixing the serial stream rate control @@ -1026,7 +1029,7 @@ intervalsOf => Double -> Fold m a b -> t m a -> t m b intervalsOf n f xs = splitWithSuffix isNothing (FL.catMaybes f) - (interjectSuffix n (return Nothing) (Serial.map Just xs)) + (interjectSuffix n (return Nothing) (map Just xs)) ------------------------------------------------------------------------------ -- Windowed classification @@ -1223,7 +1226,7 @@ classifySessionsBy tick reset ejectPred tmout , sessionCount = 0 , sessionTimerHeap = H.empty , sessionKeyValueMap = Map.empty - , sessionOutputStream = K.nil + , sessionOutputStream = IsStream.nil } -- We can eject sessions based on the current session count to limit @@ -1274,7 +1277,7 @@ classifySessionsBy tick reset ejectPred tmout let acc = Tuple' timestamp fs1 (hp1, mp1, out1, cnt1) <- do let vars = (sessionTimerHeap, sessionKeyValueMap, - K.nil, sessionCount) + IsStream.nil, sessionCount) case mOld of -- inserting new entry Nothing -> do @@ -1323,7 +1326,7 @@ classifySessionsBy tick reset ejectPred tmout ejectAll ( sessionTimerHeap , sessionKeyValueMap - , K.nil + , IsStream.nil , sessionCount ) return $ session @@ -1336,7 +1339,7 @@ classifySessionsBy tick reset ejectPred tmout -- delete from map and output the fold accumulator ejectEntry hp mp out cnt acc key = do sess <- extract acc - let out1 = (key, sess) `K.cons` out + let out1 = (key, sess) `cons` out let mp1 = Map.delete key mp return (hp, mp1, out1, cnt - 1) @@ -1372,7 +1375,8 @@ classifySessionsBy tick reset ejectPred tmout ejectExpired session@SessionState{..} curTime = do (hp', mp', out, count) <- - ejectLoop sessionTimerHeap sessionKeyValueMap K.nil sessionCount + ejectLoop + sessionTimerHeap sessionKeyValueMap IsStream.nil sessionCount return $ session { sessionCurTime = curTime , sessionCount = count @@ -1414,7 +1418,7 @@ classifySessionsBy tick reset ejectPred tmout return (hp, mp, out, cnt) -- merge timer events in the stream - stream = Serial.map Just str `Par.parallelFst` repeatM timer + stream = map Just str `parallelFst` repeatM timer timer = do liftIO $ threadDelay (round $ tick * 1000000) return Nothing @@ -1510,7 +1514,7 @@ splitInnerBy -> t m (f a) -> t m (f a) splitInnerBy splitter joiner xs = - D.fromStreamD $ D.splitInnerBy splitter joiner $ D.toStreamD xs + fromStreamD $ D.splitInnerBy splitter joiner $ toStreamD xs -- | Like 'splitInnerBy' but splits assuming the separator joins the segment in -- a suffix style. @@ -1524,4 +1528,4 @@ splitInnerBySuffix -> t m (f a) -> t m (f a) splitInnerBySuffix splitter joiner xs = - D.fromStreamD $ D.splitInnerBySuffix splitter joiner $ D.toStreamD xs + fromStreamD $ D.splitInnerBySuffix splitter joiner $ toStreamD xs diff --git a/src/Streamly/Internal/Data/Stream/IsStream/Top.hs b/src/Streamly/Internal/Data/Stream/IsStream/Top.hs index 3646c5023d..0c598ce355 100644 --- a/src/Streamly/Internal/Data/Stream/IsStream/Top.hs +++ b/src/Streamly/Internal/Data/Stream/IsStream/Top.hs @@ -61,9 +61,9 @@ import Data.Semigroup (Semigroup(..)) #endif import Streamly.Internal.Control.Concurrent (MonadAsync) import Streamly.Internal.Data.Stream.IsStream.Common (concatM) -import Streamly.Internal.Data.Stream.Prelude (foldl', fromList) +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream(..), adapt, foldl', fromList) import Streamly.Internal.Data.Stream.Serial (SerialT) -import Streamly.Internal.Data.Stream.StreamK (IsStream) import Streamly.Internal.Data.Time.Units (NanoSecond64(..), toRelTime64) import qualified Data.List as List @@ -75,7 +75,7 @@ import qualified Streamly.Internal.Data.Stream.IsStream.Generate as Stream import qualified Streamly.Internal.Data.Stream.IsStream.Expand as Stream import qualified Streamly.Internal.Data.Stream.IsStream.Reduce as Stream import qualified Streamly.Internal.Data.Stream.IsStream.Transform as Stream -import qualified Streamly.Internal.Data.Stream.StreamK as StreamK +import qualified Streamly.Internal.Data.Stream.IsStream.Type as IsStream import Prelude hiding (filter, zipWith, concatMap, concat) @@ -272,7 +272,7 @@ innerJoin eq s1 s2 = do b <- s2 if a `eq` b then return (a, b) - else StreamK.nil + else Stream.nil -- If the second stream is too big it can be partitioned based on hashes and -- then we can process one parition at a time. @@ -338,8 +338,8 @@ leftJoin eq s1 s2 = Stream.evalStateT (return False) $ do let final = do r <- lift get if r - then StreamK.nil - else StreamK.fromPure Nothing + then Stream.nil + else Stream.fromPure Nothing b <- fmap Just (Stream.liftInner s2) <> final case b of Just b1 -> @@ -347,7 +347,7 @@ leftJoin eq s1 s2 = Stream.evalStateT (return False) $ do then do lift $ put True return (a, Just b1) - else StreamK.nil + else Stream.nil Nothing -> return (a, Nothing) -- | Like 'outerJoin' but uses a hashmap for efficiency. @@ -415,11 +415,11 @@ outerJoin eq s1 s = let final = do r <- lift get if r - then StreamK.nil - else StreamK.fromPure Nothing + then Stream.nil + else Stream.fromPure Nothing (_i, b) <- - Stream.indexed - $ fmap Just (Stream.liftInner (Array.toStream arr)) <> final + let stream = IsStream.fromSerial $ Array.toStream arr + in Stream.indexed $ fmap Just (Stream.liftInner stream) <> final case b of Just (b1, _used) -> if a `eq` b1 @@ -428,7 +428,7 @@ outerJoin eq s1 s = -- XXX Need to use a mutable array -- when (not used) $ Array.writeIndex i True return (Just a, Just b1) - else StreamK.nil + else Stream.nil Nothing -> return (Just a, Nothing) -- Put the b's that have been paired, in another hash or mutate the hash to set @@ -496,7 +496,7 @@ intersectBy eq s1 s2 = concatM $ do -- This may work well when s2 is small - xs <- Stream.toListRev $ Stream.uniqBy eq $ StreamK.adapt s2 + xs <- Stream.toListRev $ Stream.uniqBy eq $ adapt s2 return $ Stream.filter (\x -> List.any (eq x) xs) s1 -- | Like 'intersectBy' but works only on sorted streams. @@ -546,7 +546,7 @@ differenceBy eq s1 s2 = -- not emitting an element if it was successfully deleted from s2. -- we will need a deleteBy that can return whether the element was -- deleted or not. - xs <- Stream.toList $ StreamK.adapt s1 + xs <- Stream.toList $ adapt s1 fmap fromList $ foldl' (flip (List.deleteBy eq)) xs s2 -- | Like 'differenceBy' but works only on sorted streams. @@ -585,7 +585,7 @@ unionBy :: (IsStream t, MonadAsync m, Semigroup (t m a)) => unionBy eq s1 s2 = concatM $ do - xs <- Stream.toList $ StreamK.adapt s2 + xs <- Stream.toList $ adapt s2 -- XXX we can use postscanlMAfter' instead of IORef ref <- liftIO $ newIORef $! List.nubBy eq xs let f x = do diff --git a/src/Streamly/Internal/Data/Stream/IsStream/Transform.hs b/src/Streamly/Internal/Data/Stream/IsStream/Transform.hs index 7069a46016..c8047c6f83 100644 --- a/src/Streamly/Internal/Data/Stream/IsStream/Transform.hs +++ b/src/Streamly/Internal/Data/Stream/IsStream/Transform.hs @@ -14,11 +14,12 @@ module Streamly.Internal.Data.Stream.IsStream.Transform -- * Folding , foldrS + , foldrSShared , foldrT -- * Mapping -- | Stateless one-to-one maps. - , Serial.map + , map , sequence , mapM , smapM @@ -30,6 +31,8 @@ module Streamly.Internal.Data.Stream.IsStream.Transform , tap , tapOffsetEvery , tapAsync + , tapAsyncK + , distributeAsync_ , tapRate , pollCounts @@ -170,7 +173,8 @@ module Streamly.Internal.Data.Stream.IsStream.Transform -- ** Concurrent Pipelines -- | Run streaming stages concurrently. - , Par.mkParallel + , mkAsync + , mkParallel , applyAsync , (|$) , (|&) @@ -226,6 +230,7 @@ import Streamly.Internal.Data.Stream.IsStream.Common ( absTimesWith , drop , findIndices + , map , postscanlM' , relTimesWith , reverse @@ -236,31 +241,35 @@ import Streamly.Internal.Data.Stream.IsStream.Common , takeWhile , interjectSuffix , intersperseM + , mkAsync + , mkParallel + , zipWith ) import Streamly.Internal.Control.Concurrent (MonadAsync) -import Streamly.Internal.Data.Stream.Prelude (fromStreamS, toStreamS) +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream(..), fromStreamS, toStreamS, fromStreamD, toStreamD, toConsK) import Streamly.Internal.Data.Stream.Serial (SerialT) -import Streamly.Internal.Data.Stream.StreamD (fromStreamD, toStreamD) -import Streamly.Internal.Data.Stream.StreamK (IsStream) import Streamly.Internal.Data.SVar (Rate(..)) import Streamly.Internal.Data.Time.Units (TimeUnit64, AbsTime, RelTime64) import qualified Streamly.Internal.Data.Fold as FL import qualified Streamly.Internal.Data.Stream.Parallel as Par -import qualified Streamly.Internal.Data.Stream.Prelude as P import qualified Streamly.Internal.Data.Stream.Serial as Serial import qualified Streamly.Internal.Data.Stream.StreamD as D +#if __GLASGOW_HASKELL__ == 802 import qualified Streamly.Internal.Data.Stream.StreamK as K -import qualified Streamly.Internal.Data.Stream.Zip as Z +#endif +import qualified Streamly.Internal.Data.Stream.StreamK.Type as K #ifdef USE_STREAMK_ONLY import qualified Streamly.Internal.Data.Stream.StreamK as S #else import qualified Streamly.Internal.Data.Stream.StreamD as S #endif +import qualified Prelude import Prelude hiding ( filter, drop, dropWhile, take, takeWhile, foldr, map, mapM, sequence - , reverse, foldr1 , scanl, scanl1) + , reverse, foldr1 , scanl, scanl1, zipWith) -- -- $setup @@ -320,7 +329,21 @@ transform pipe xs = fromStreamD $ D.transform pipe (toStreamD xs) -- /Pre-release/ {-# INLINE foldrS #-} foldrS :: IsStream t => (a -> t m b -> t m b) -> t m b -> t m a -> t m b -foldrS = K.foldrS +foldrS f z xs = + fromStream + $ K.foldrS + (\y ys -> toStream $ f y (fromStream ys)) + (toStream z) + (toStream xs) + +{-# INLINE foldrSShared #-} +foldrSShared :: IsStream t => (a -> t m b -> t m b) -> t m b -> t m a -> t m b +foldrSShared f z xs = + fromStream + $ K.foldrSShared + (\y ys -> toStream $ f y (fromStream ys)) + (toStream z) + (toStream xs) -- | Right fold to a transformer monad. This is the most general right fold -- function. 'foldrS' is a special case of 'foldrT', however 'foldrS' @@ -370,8 +393,9 @@ foldrT f z s = S.foldrT f z (toStreamS s) -- -- @since 0.1.0 {-# INLINE_EARLY mapM #-} -mapM :: (IsStream t, MonadAsync m) => (a -> m b) -> t m a -> t m b -mapM = K.mapM +mapM :: forall t m a b. (IsStream t, MonadAsync m) => + (a -> m b) -> t m a -> t m b +mapM f = fromStream . K.mapMWith (toConsK (consM @t)) f . toStream {-# RULES "mapM serial" mapM = mapMSerial #-} {-# INLINE mapMSerial #-} @@ -439,7 +463,7 @@ sequence m = fromStreamS $ S.sequence (toStreamS m) -- @since 0.7.0 {-# INLINE tap #-} tap :: (IsStream t, Monad m) => FL.Fold m a b -> t m a -> t m a -tap f xs = D.fromStreamD $ D.tap f (D.toStreamD xs) +tap f xs = fromStreamD $ D.tap f (toStreamD xs) -- XXX Remove this. It can be expressed in terms of Fold.sampleFromThen. -- @@ -458,7 +482,7 @@ tap f xs = D.fromStreamD $ D.tap f (D.toStreamD xs) tapOffsetEvery :: (IsStream t, Monad m) => Int -> Int -> FL.Fold m a b -> t m a -> t m a tapOffsetEvery offset n f xs = - D.fromStreamD $ D.tapOffsetEvery offset n f (D.toStreamD xs) + fromStreamD $ D.tapOffsetEvery offset n f (toStreamD xs) -- | Redirect a copy of the stream to a supplied fold and run it concurrently -- in an independent thread. The fold may buffer some elements. The buffer size @@ -490,7 +514,61 @@ tapOffsetEvery offset n f xs = -- /Pre-release/ {-# INLINE tapAsync #-} tapAsync :: (IsStream t, MonadAsync m) => FL.Fold m a b -> t m a -> t m a -tapAsync f xs = D.fromStreamD $ Par.tapAsyncF f (D.toStreamD xs) +tapAsync f xs = fromStreamD $ Par.tapAsyncF f (toStreamD xs) + +-- | Redirect a copy of the stream to a supplied fold and run it concurrently +-- in an independent thread. The fold may buffer some elements. The buffer size +-- is determined by the prevailing 'Streamly.Prelude.maxBuffer' setting. +-- +-- @ +-- Stream m a -> m b +-- | +-- -----stream m a ---------------stream m a----- +-- +-- @ +-- +-- @ +-- > S.drain $ S.tapAsync (S.mapM_ print) (S.enumerateFromTo 1 2) +-- 1 +-- 2 +-- @ +-- +-- Exceptions from the concurrently running fold are propagated to the current +-- computation. Note that, because of buffering in the fold, exceptions may be +-- delayed and may not correspond to the current element being processed in the +-- parent stream, but we guarantee that before the parent stream stops the tap +-- finishes and all exceptions from it are drained. +-- +-- +-- Compare with 'tap'. +-- +-- /Pre-release/ +{-# INLINE tapAsyncK #-} +tapAsyncK :: (IsStream t, MonadAsync m) => (t m a -> m b) -> t m a -> t m a +tapAsyncK f m = fromStream $ Par.tapAsyncK (f . fromStream) (toStream m) + +-- | Concurrently distribute a stream to a collection of fold functions, +-- discarding the outputs of the folds. +-- +-- @ +-- > Stream.drain $ Stream.distributeAsync_ [Stream.mapM_ print, Stream.mapM_ print] (Stream.enumerateFromTo 1 2) +-- 1 +-- 2 +-- 1 +-- 2 +-- +-- @ +-- +-- @ +-- distributeAsync_ = flip (foldr tapAsync) +-- @ +-- +-- /Pre-release/ +-- +{-# INLINE distributeAsync_ #-} +distributeAsync_ :: (Foldable f, IsStream t, MonadAsync m) + => f (t m a -> m b) -> t m a -> t m a +distributeAsync_ = flip (Prelude.foldr tapAsyncK) -- | @pollCounts predicate transform fold stream@ counts those elements in the -- stream that pass the @predicate@. The resulting count stream is sent to @@ -518,9 +596,9 @@ pollCounts :: -> t m a -> t m a pollCounts predicate transf f xs = - D.fromStreamD - $ D.pollCounts predicate (D.toStreamD . transf . D.fromStreamD) f - $ D.toStreamD xs + fromStreamD + $ D.pollCounts predicate (toStreamD . transf . fromStreamD) f + $ toStreamD xs -- | Calls the supplied function with the number of elements consumed -- every @n@ seconds. The given function is run in a separate thread @@ -547,7 +625,7 @@ tapRate :: -> (Int -> m b) -> t m a -> t m a -tapRate n f xs = D.fromStreamD $ D.tapRate n f $ D.toStreamD xs +tapRate n f xs = fromStreamD $ D.tapRate n f $ toStreamD xs -- | Apply a monadic function to each element flowing through the stream and -- discard the results. @@ -583,7 +661,7 @@ trace f = mapM (\x -> void (f x) >> return x) -- /Pre-release/ {-# INLINE trace_ #-} trace_ :: (IsStream t, Monad m) => m b -> t m a -> t m a -trace_ eff = Serial.mapM (\x -> eff >> return x) +trace_ eff = fromStreamD . D.mapM (\x -> eff >> return x) . toStreamD ------------------------------------------------------------------------------ -- Scanning with a Fold @@ -597,7 +675,8 @@ trace_ eff = Serial.mapM (\x -> eff >> return x) -- @since 0.7.0 {-# INLINE scan #-} scan :: (IsStream t, Monad m) => Fold m a b -> t m a -> t m b -scan = P.scanOnce +-- scan = P.scanOnce +scan fld m = fromStreamD $ D.scanOnce fld $ toStreamD m -- | Postscan a stream using the given monadic fold. -- @@ -617,7 +696,7 @@ scan = P.scanOnce -- @since 0.7.0 {-# INLINE postscan #-} postscan :: (IsStream t, Monad m) => Fold m a b -> t m a -> t m b -postscan = P.postscanOnce +postscan fld = fromStreamD . D.postscanOnce fld . toStreamD ------------------------------------------------------------------------------ -- Scanning - Transformation by Folding @@ -644,7 +723,7 @@ postscan = P.postscanOnce {-# DEPRECATED scanx "Please use scanl followed by map instead." #-} {-# INLINE scanx #-} scanx :: (IsStream t, Monad m) => (x -> a -> x) -> x -> (x -> b) -> t m a -> t m b -scanx = P.scanlx' +scanx step begin done = fromStreamS . S.scanlx' step begin done . toStreamS -- XXX this needs to be concurrent -- XXX because of the use of D.cons for appending, scanlM' has quadratic @@ -807,7 +886,7 @@ filter :: (IsStream t, Monad m) => (a -> Bool) -> t m a -> t m a filter p m = fromStreamS $ S.filter p $ toStreamS m #else filter :: IsStream t => (a -> Bool) -> t m a -> t m a -filter = K.filter +filter p m = fromStream $ K.filter p $ toStream m #endif -- | Same as 'filter' but with a monadic predicate. @@ -1365,7 +1444,7 @@ indexedR n = fromStreamD . D.indexedR n . toStreamD {-# INLINE timestampWith #-} timestampWith :: (IsStream t, MonadAsync m, Functor (t m)) => Double -> t m a -> t m (AbsTime, a) -timestampWith g stream = Z.zipWith (flip (,)) stream (absTimesWith g) +timestampWith g stream = zipWith (flip (,)) stream (absTimesWith g) -- TBD: check performance vs a custom implementation without using zipWith. -- @@ -1390,7 +1469,7 @@ timestamped = timestampWith 0.01 {-# INLINE timeIndexWith #-} timeIndexWith :: (IsStream t, MonadAsync m, Functor (t m)) => Double -> t m a -> t m (RelTime64, a) -timeIndexWith g stream = Z.zipWith (flip (,)) stream (relTimesWith g) +timeIndexWith g stream = zipWith (flip (,)) stream (relTimesWith g) -- | Pair each element in a stream with relative times starting from 0, using a -- 10 ms granularity clock. The time is measured just before the element is @@ -1480,7 +1559,7 @@ mapMaybe f m = fromStreamS $ S.mapMaybe f $ toStreamS m {-# INLINE_EARLY mapMaybeM #-} mapMaybeM :: (IsStream t, MonadAsync m, Functor (t m)) => (a -> m (Maybe b)) -> t m a -> t m b -mapMaybeM f = fmap fromJust . filter isJust . K.mapM f +mapMaybeM f = fmap fromJust . filter isJust . mapM f {-# RULES "mapMaybeM serial" mapMaybeM = mapMaybeMSerial #-} {-# INLINE mapMaybeMSerial #-} @@ -1559,8 +1638,8 @@ both = fmap (either id id) -- @since 0.8.0 {-# INLINE (|$) #-} (|$) :: (IsStream t, MonadAsync m) => (t m a -> t m b) -> (t m a -> t m b) --- (|$) f = f . Async.mkAsync -(|$) f = f . Par.mkParallel +-- (|$) f = f . mkAsync +(|$) f = f . mkParallel infixr 0 |$ diff --git a/src/Streamly/Internal/Data/Stream/IsStream/Type.hs b/src/Streamly/Internal/Data/Stream/IsStream/Type.hs new file mode 100644 index 0000000000..8acebae440 --- /dev/null +++ b/src/Streamly/Internal/Data/Stream/IsStream/Type.hs @@ -0,0 +1,698 @@ +{-# LANGUAGE UndecidableInstances #-} +{-# OPTIONS_GHC -Wno-orphans #-} + +-- | +-- Module : Streamly.Internal.Data.Stream.Type +-- Copyright : (c) 2017 Composewell Technologies +-- License : BSD3 +-- Maintainer : streamly@composewell.com +-- Stability : experimental +-- Portability : GHC +-- +-- +module Streamly.Internal.Data.Stream.IsStream.Type + ( + -- * IsStream Type Class + IsStream (..) + , K.Stream (..) + , Streaming + + -- * Type Conversion + , fromStreamS + , toStreamS + , fromStreamD + , toStreamD + , adapt + , toConsK + + -- * Building a stream + , mkStream + , foldStreamShared + , foldStream + + -- * Stream Types + , SerialT + , Serial + , fromSerial + + , WSerialT + , WSerial + , fromWSerial + + , AsyncT + , Async + , fromAsync + + , WAsyncT + , WAsync + , fromWAsync + + , AheadT + , Ahead + , fromAhead + + , ParallelT + , Parallel + , fromParallel + + , ZipSerialM + , ZipSerial + , fromZipSerial + + , ZipAsyncM + , ZipAsync + , fromZipAsync + + -- * Construction + , cons + , (.:) + , nil + , nilM + , fromPure + , fromEffect + , repeat + + -- * Bind/Concat + , bindWith + , concatMapWith + + -- * Fold Utilities + , concatFoldableWith + , concatMapFoldableWith + , concatForFoldableWith + + -- * Running Effects + , drain + + -- * Conversion operations + , fromList + , toList + + -- * Fold operations + , foldrM + , foldrMx + , foldr + + , foldlx' + , foldlMx' + , foldl' + , fold + + -- * Zip style operations + , eqBy + , cmpBy + + -- * Deprecated + , interleaving + , zipping + , zippingAsync + ) +where + +import Streamly.Internal.Control.Concurrent (MonadAsync) +import Streamly.Internal.Data.Fold.Type (Fold (..)) +import Streamly.Internal.Data.Stream.Serial + (SerialT(..), Serial, WSerialT(..), WSerial) +import Streamly.Internal.Data.Stream.Async + (AsyncT(..), Async, WAsyncT(..), WAsync) +import Streamly.Internal.Data.Stream.Ahead (AheadT(..), Ahead) +import Streamly.Internal.Data.Stream.Parallel (ParallelT(..), Parallel) +import Streamly.Internal.Data.Stream.Zip + (ZipSerialM(..), ZipSerial, ZipAsyncM(..), ZipAsync) +import Streamly.Internal.Data.SVar.Type (State, adaptState) + +import qualified Prelude +import qualified Streamly.Internal.Data.Stream.Ahead as Ahead +import qualified Streamly.Internal.Data.Stream.Async as Async +import qualified Streamly.Internal.Data.Stream.Parallel as Parallel +import qualified Streamly.Internal.Data.Stream.Serial as Serial +import qualified Streamly.Internal.Data.Stream.StreamD.Type as D +import qualified Streamly.Internal.Data.Stream.StreamK.Type as K +#ifdef USE_STREAMK_ONLY +import qualified Streamly.Internal.Data.Stream.StreamK as S +import qualified Streamly.Internal.Data.Stream.StreamK.Type as S +#else +import qualified Streamly.Internal.Data.Stream.StreamD.Type as S +#endif +import qualified Streamly.Internal.Data.Stream.Zip as Zip + +import Prelude hiding (foldr, repeat) + +#include "inline.hs" +#define USE_IS_STREAM +#include "PreludeCommon.hs" + +------------------------------------------------------------------------------ +-- Types that can behave as a Stream +------------------------------------------------------------------------------ + +infixr 5 `consM` +infixr 5 |: + +-- XXX Use a different SVar based on the stream type. But we need to make sure +-- that we do not lose performance due to polymorphism. +-- +-- | Class of types that can represent a stream of elements of some type 'a' in +-- some monad 'm'. +-- +-- /Since: 0.2.0 ("Streamly")/ +-- +-- @since 0.8.0 +class +#if __GLASGOW_HASKELL__ >= 806 + ( forall m a. MonadAsync m => Semigroup (t m a) + , forall m a. MonadAsync m => Monoid (t m a) + , forall m. Monad m => Functor (t m) + , forall m. MonadAsync m => Applicative (t m) + ) => +#endif + IsStream t where + toStream :: t m a -> K.Stream m a + fromStream :: K.Stream m a -> t m a + -- | Constructs a stream by adding a monadic action at the head of an + -- existing stream. For example: + -- + -- @ + -- > toList $ getLine \`consM` getLine \`consM` nil + -- hello + -- world + -- ["hello","world"] + -- @ + -- + -- /Concurrent (do not use 'fromParallel' to construct infinite streams)/ + -- + -- @since 0.2.0 + consM :: MonadAsync m => m a -> t m a -> t m a + -- | Operator equivalent of 'consM'. We can read it as "@parallel colon@" + -- to remember that @|@ comes before ':'. + -- + -- @ + -- > toList $ getLine |: getLine |: nil + -- hello + -- world + -- ["hello","world"] + -- @ + -- + -- @ + -- let delay = threadDelay 1000000 >> print 1 + -- drain $ fromSerial $ delay |: delay |: delay |: nil + -- drain $ fromParallel $ delay |: delay |: delay |: nil + -- @ + -- + -- /Concurrent (do not use 'fromParallel' to construct infinite streams)/ + -- + -- @since 0.2.0 + (|:) :: MonadAsync m => m a -> t m a -> t m a + -- We can define (|:) just as 'consM' but it is defined explicitly for each + -- type because we want to use SPECIALIZE pragma on the definition. + +-- | Same as 'IsStream'. +-- +-- @since 0.1.0 +{-# DEPRECATED Streaming "Please use IsStream instead." #-} +type Streaming = IsStream + +------------------------------------------------------------------------------- +-- Type adapting combinators +------------------------------------------------------------------------------- + +-- XXX Move/reset the State here by reconstructing the stream with cleared +-- state. Can we make sure we do not do that when t1 = t2? If we do this then +-- we do not need to do that explicitly using svarStyle. It would act as +-- unShare when the stream type is the same. +-- +-- | Adapt any specific stream type to any other specific stream type. +-- +-- /Since: 0.1.0 ("Streamly")/ +-- +-- @since 0.8.0 +adapt :: (IsStream t1, IsStream t2) => t1 m a -> t2 m a +adapt = fromStream . toStream + +{-# INLINE fromStreamD #-} +fromStreamD :: (IS_STREAM Monad m) => D.Stream m a -> STREAM m a +fromStreamD = FROM_STREAM . D.toStreamK + +-- | Adapt a polymorphic consM operation to a StreamK cons operation +{-# INLINE toConsK #-} +toConsK :: IsStream t => + (m a -> t m a -> t m a) -> m a -> K.Stream m a -> K.Stream m a +toConsK cns x xs = toStream $ x `cns` fromStream xs + +------------------------------------------------------------------------------ +-- Building a stream +------------------------------------------------------------------------------ + +-- XXX The State is always parameterized by "Stream" which means State is not +-- different for different stream types. So we have to manually make sure that +-- when converting from one stream to another we migrate the state correctly. +-- This can be fixed if we use a different SVar type for different streams. +-- Currently we always use "SVar Stream" and therefore a different State type +-- parameterized by that stream. +-- +-- XXX Since t is coercible we should be able to coerce k +-- mkStream k = fromStream $ MkStream $ coerce k +-- +-- | Build a stream from an 'SVar', a stop continuation, a singleton stream +-- continuation and a yield continuation. +{-# INLINE_EARLY mkStream #-} +mkStream :: IsStream t + => (forall r. State K.Stream m a + -> (a -> t m a -> m r) + -> (a -> m r) + -> m r + -> m r) + -> t m a +mkStream k = fromStream $ K.MkStream $ \st yld sng stp -> + let yieldk a r = yld a (toStream r) + in k st yieldk sng stp + +{-# RULES "mkStream from stream" mkStream = mkStreamFromStream #-} +mkStreamFromStream :: IsStream t + => (forall r. State K.Stream m a + -> (a -> K.Stream m a -> m r) + -> (a -> m r) + -> m r + -> m r) + -> t m a +mkStreamFromStream k = fromStream $ K.MkStream k + +{-# RULES "mkStream stream" mkStream = mkStreamStream #-} +mkStreamStream + :: (forall r. State K.Stream m a + -> (a -> K.Stream m a -> m r) + -> (a -> m r) + -> m r + -> m r) + -> K.Stream m a +mkStreamStream = K.MkStream + +------------------------------------------------------------------------------ +-- Folding a stream +------------------------------------------------------------------------------ + +-- | Fold a stream by providing an SVar, a stop continuation, a singleton +-- continuation and a yield continuation. The stream would share the current +-- SVar passed via the State. +{-# INLINE_EARLY foldStreamShared #-} +foldStreamShared + :: IsStream t + => State K.Stream m a + -> (a -> t m a -> m r) + -> (a -> m r) + -> m r + -> t m a + -> m r +foldStreamShared st yld sng stp m = + let yieldk a x = yld a (fromStream x) + K.MkStream k = toStream m + in k st yieldk sng stp + +-- XXX write a similar rule for foldStream as well? +{-# RULES "foldStreamShared from stream" + foldStreamShared = foldStreamSharedStream #-} +foldStreamSharedStream + :: State K.Stream m a + -> (a -> K.Stream m a -> m r) + -> (a -> m r) + -> m r + -> K.Stream m a + -> m r +foldStreamSharedStream st yld sng stp m = + let K.MkStream k = m + in k st yld sng stp + +-- | Fold a stream by providing a State, stop continuation, a singleton +-- continuation and a yield continuation. The stream will not use the SVar +-- passed via State. +{-# INLINE foldStream #-} +foldStream + :: IsStream t + => State K.Stream m a + -> (a -> t m a -> m r) + -> (a -> m r) + -> m r + -> t m a + -> m r +foldStream st yld sng stp m = + let yieldk a x = yld a (fromStream x) + K.MkStream k = toStream m + in k (adaptState st) yieldk sng stp + +------------------------------------------------------------------------------- +-- Serial +------------------------------------------------------------------------------- + +-- | Fix the type of a polymorphic stream as 'SerialT'. +-- +-- /Since: 0.1.0 ("Streamly")/ +-- +-- @since 0.8.0 +fromSerial :: IsStream t => SerialT m a -> t m a +fromSerial = adapt + +instance IsStream SerialT where + toStream = getSerialT + fromStream = SerialT + + {-# INLINE consM #-} + {-# SPECIALIZE consM :: IO a -> SerialT IO a -> SerialT IO a #-} + consM = Serial.consM + + {-# INLINE (|:) #-} + {-# SPECIALIZE (|:) :: IO a -> SerialT IO a -> SerialT IO a #-} + (|:) = Serial.consM + +-- | Fix the type of a polymorphic stream as 'WSerialT'. +-- +-- /Since: 0.2.0 ("Streamly")/ +-- +-- @since 0.8.0 +fromWSerial :: IsStream t => WSerialT m a -> t m a +fromWSerial = adapt + +-- | Same as 'fromWSerial'. +-- +-- @since 0.1.0 +{-# DEPRECATED interleaving "Please use fromWSerial instead." #-} +interleaving :: IsStream t => WSerialT m a -> t m a +interleaving = fromWSerial + +instance IsStream WSerialT where + toStream = getWSerialT + fromStream = WSerialT + + {-# INLINE consM #-} + {-# SPECIALIZE consM :: IO a -> WSerialT IO a -> WSerialT IO a #-} + consM :: Monad m => m a -> WSerialT m a -> WSerialT m a + consM = Serial.consMWSerial + + {-# INLINE (|:) #-} + {-# SPECIALIZE (|:) :: IO a -> WSerialT IO a -> WSerialT IO a #-} + (|:) :: Monad m => m a -> WSerialT m a -> WSerialT m a + (|:) = Serial.consMWSerial + +------------------------------------------------------------------------------- +-- Async +------------------------------------------------------------------------------- + +-- | Fix the type of a polymorphic stream as 'AsyncT'. +-- +-- /Since: 0.1.0 ("Streamly")/ +-- +-- @since 0.8.0 +fromAsync :: IsStream t => AsyncT m a -> t m a +fromAsync = adapt + +instance IsStream AsyncT where + toStream = getAsyncT + fromStream = AsyncT + + {-# INLINE consM #-} + {-# SPECIALIZE consM :: IO a -> AsyncT IO a -> AsyncT IO a #-} + consM = Async.consMAsync + + {-# INLINE (|:) #-} + {-# SPECIALIZE (|:) :: IO a -> AsyncT IO a -> AsyncT IO a #-} + (|:) = Async.consMAsync + +-- | Fix the type of a polymorphic stream as 'WAsyncT'. +-- +-- /Since: 0.2.0 ("Streamly")/ +-- +-- @since 0.8.0 +fromWAsync :: IsStream t => WAsyncT m a -> t m a +fromWAsync = adapt + +instance IsStream WAsyncT where + toStream = getWAsyncT + fromStream = WAsyncT + + {-# INLINE consM #-} + {-# SPECIALIZE consM :: IO a -> WAsyncT IO a -> WAsyncT IO a #-} + consM = Async.consMWAsync + + {-# INLINE (|:) #-} + {-# SPECIALIZE (|:) :: IO a -> WAsyncT IO a -> WAsyncT IO a #-} + (|:) = Async.consMWAsync + +------------------------------------------------------------------------------- +-- Ahead +------------------------------------------------------------------------------- + +-- | Fix the type of a polymorphic stream as 'AheadT'. +-- +-- /Since: 0.3.0 ("Streamly")/ +-- +-- @since 0.8.0 +fromAhead :: IsStream t => AheadT m a -> t m a +fromAhead = adapt + +instance IsStream AheadT where + toStream = getAheadT + fromStream = AheadT + + {-# INLINE consM #-} + {-# SPECIALIZE consM :: IO a -> AheadT IO a -> AheadT IO a #-} + consM = Ahead.consM + + {-# INLINE (|:) #-} + {-# SPECIALIZE (|:) :: IO a -> AheadT IO a -> AheadT IO a #-} + (|:) = Ahead.consM + +------------------------------------------------------------------------------- +-- Parallel +------------------------------------------------------------------------------- + +-- | Fix the type of a polymorphic stream as 'ParallelT'. +-- +-- /Since: 0.1.0 ("Streamly")/ +-- +-- @since 0.8.0 +fromParallel :: IsStream t => ParallelT m a -> t m a +fromParallel = adapt + +instance IsStream ParallelT where + toStream = getParallelT + fromStream = ParallelT + + {-# INLINE consM #-} + {-# SPECIALIZE consM :: IO a -> ParallelT IO a -> ParallelT IO a #-} + consM = Parallel.consM + + {-# INLINE (|:) #-} + {-# SPECIALIZE (|:) :: IO a -> ParallelT IO a -> ParallelT IO a #-} + (|:) = Parallel.consM + +------------------------------------------------------------------------------- +-- Zip +------------------------------------------------------------------------------- + +-- | Fix the type of a polymorphic stream as 'ZipSerialM'. +-- +-- /Since: 0.2.0 ("Streamly")/ +-- +-- @since 0.8.0 +fromZipSerial :: IsStream t => ZipSerialM m a -> t m a +fromZipSerial = adapt + +-- | Same as 'fromZipSerial'. +-- +-- @since 0.1.0 +{-# DEPRECATED zipping "Please use fromZipSerial instead." #-} +zipping :: IsStream t => ZipSerialM m a -> t m a +zipping = fromZipSerial + +instance IsStream ZipSerialM where + toStream = getZipSerialM + fromStream = ZipSerialM + + {-# INLINE consM #-} + {-# SPECIALIZE consM :: IO a -> ZipSerialM IO a -> ZipSerialM IO a #-} + consM :: Monad m => m a -> ZipSerialM m a -> ZipSerialM m a + consM = Zip.consMZip + + {-# INLINE (|:) #-} + {-# SPECIALIZE (|:) :: IO a -> ZipSerialM IO a -> ZipSerialM IO a #-} + (|:) :: Monad m => m a -> ZipSerialM m a -> ZipSerialM m a + (|:) = Zip.consMZip + +-- | Fix the type of a polymorphic stream as 'ZipAsyncM'. +-- +-- /Since: 0.2.0 ("Streamly")/ +-- +-- @since 0.8.0 +fromZipAsync :: IsStream t => ZipAsyncM m a -> t m a +fromZipAsync = adapt + +-- | Same as 'fromZipAsync'. +-- +-- @since 0.1.0 +{-# DEPRECATED zippingAsync "Please use fromZipAsync instead." #-} +zippingAsync :: IsStream t => ZipAsyncM m a -> t m a +zippingAsync = fromZipAsync + +instance IsStream ZipAsyncM where + toStream = getZipAsyncM + fromStream = ZipAsyncM + + {-# INLINE consM #-} + {-# SPECIALIZE consM :: IO a -> ZipAsyncM IO a -> ZipAsyncM IO a #-} + consM :: Monad m => m a -> ZipAsyncM m a -> ZipAsyncM m a + consM = Zip.consMZipAsync + + {-# INLINE (|:) #-} + {-# SPECIALIZE (|:) :: IO a -> ZipAsyncM IO a -> ZipAsyncM IO a #-} + (|:) :: Monad m => m a -> ZipAsyncM m a -> ZipAsyncM m a + (|:) = Zip.consMZipAsync + +------------------------------------------------------------------------------- +-- Construction +------------------------------------------------------------------------------- + +infixr 5 `cons` + +-- | Construct a stream by adding a pure value at the head of an existing +-- stream. For serial streams this is the same as @(return a) \`consM` r@ but +-- more efficient. For concurrent streams this is not concurrent whereas +-- 'consM' is concurrent. For example: +-- +-- @ +-- > toList $ 1 \`cons` 2 \`cons` 3 \`cons` nil +-- [1,2,3] +-- @ +-- +-- @since 0.1.0 +{-# INLINE_NORMAL cons #-} +cons :: IsStream t => a -> t m a -> t m a +cons a r = fromStream $ K.cons a (toStream r) + +infixr 5 .: + +-- | Operator equivalent of 'cons'. +-- +-- @ +-- > toList $ 1 .: 2 .: 3 .: nil +-- [1,2,3] +-- @ +-- +-- @since 0.1.1 +{-# INLINE (.:) #-} +(.:) :: IsStream t => a -> t m a -> t m a +(.:) = cons + +{-# INLINE_NORMAL nil #-} +nil :: IsStream t => t m a +nil = fromStream K.nil + +{-# INLINE_NORMAL nilM #-} +nilM :: (IsStream t, Monad m) => m b -> t m a +nilM = fromStream . K.nilM + +{-# INLINE_NORMAL fromPure #-} +fromPure :: IsStream t => a -> t m a +fromPure = fromStream . K.fromPure + +{-# INLINE_NORMAL fromEffect #-} +fromEffect :: (Monad m, IsStream t) => m a -> t m a +fromEffect = fromStream . K.fromEffect + +{-# INLINE repeat #-} +repeat :: IsStream t => a -> t m a +repeat = fromStream . K.repeat + +------------------------------------------------------------------------------- +-- Bind/Concat +------------------------------------------------------------------------------- + +{-# INLINE bindWith #-} +bindWith + :: IsStream t + => (t m b -> t m b -> t m b) + -> t m a + -> (a -> t m b) + -> t m b +bindWith par m1 f = + fromStream + $ K.bindWith + (\s1 s2 -> toStream $ par (fromStream s1) (fromStream s2)) + (toStream m1) + (toStream . f) + +-- | @concatMapWith mixer generator stream@ is a two dimensional looping +-- combinator. The @generator@ function is used to generate streams from the +-- elements in the input @stream@ and the @mixer@ function is used to merge +-- those streams. +-- +-- Note we can merge streams concurrently by using a concurrent merge function. +-- +-- /Since: 0.7.0/ +-- +-- /Since: 0.8.0 (signature change)/ +{-# INLINE concatMapWith #-} +concatMapWith + :: IsStream t + => (t m b -> t m b -> t m b) + -> (a -> t m b) + -> t m a + -> t m b +concatMapWith par f xs = bindWith par xs f + +-- | A variant of 'foldMap' that allows you to map a monadic streaming action +-- on a 'Foldable' container and then fold it using the specified stream merge +-- operation. +-- +-- @concatMapFoldableWith 'async' return [1..3]@ +-- +-- Equivalent to: +-- +-- @ +-- concatMapFoldableWith f g = Prelude.foldr (f . g) S.nil +-- concatMapFoldableWith f g xs = S.concatMapWith f g (S.fromFoldable xs) +-- @ +-- +-- /Since: 0.8.0 (Renamed foldMapWith to concatMapFoldableWith)/ +-- +-- /Since: 0.1.0 ("Streamly")/ +{-# INLINABLE concatMapFoldableWith #-} +concatMapFoldableWith :: (IsStream t, Foldable f) + => (t m b -> t m b -> t m b) -> (a -> t m b) -> f a -> t m b +concatMapFoldableWith f g = Prelude.foldr (f . g) nil + +-- | Like 'concatMapFoldableWith' but with the last two arguments reversed i.e. the +-- monadic streaming function is the last argument. +-- +-- Equivalent to: +-- +-- @ +-- concatForFoldableWith f xs g = Prelude.foldr (f . g) S.nil xs +-- concatForFoldableWith f = flip (S.concatMapFoldableWith f) +-- @ +-- +-- /Since: 0.8.0 (Renamed forEachWith to concatForFoldableWith)/ +-- +-- /Since: 0.1.0 ("Streamly")/ +{-# INLINE concatForFoldableWith #-} +concatForFoldableWith :: (IsStream t, Foldable f) + => (t m b -> t m b -> t m b) -> f a -> (a -> t m b) -> t m b +concatForFoldableWith f = flip (concatMapFoldableWith f) + +-- | A variant of 'Data.Foldable.fold' that allows you to fold a 'Foldable' +-- container of streams using the specified stream sum operation. +-- +-- @concatFoldableWith 'async' $ map return [1..3]@ +-- +-- Equivalent to: +-- +-- @ +-- concatFoldableWith f = Prelude.foldr f S.nil +-- concatFoldableWith f = S.concatMapFoldableWith f id +-- @ +-- +-- /Since: 0.8.0 (Renamed foldWith to concatFoldableWith)/ +-- +-- /Since: 0.1.0 ("Streamly")/ +{-# INLINE concatFoldableWith #-} +concatFoldableWith :: (IsStream t, Foldable f) + => (t m a -> t m a -> t m a) -> f (t m a) -> t m a +concatFoldableWith f = concatMapFoldableWith f id diff --git a/src/Streamly/Internal/Data/Stream/IsStream/Types.hs b/src/Streamly/Internal/Data/Stream/IsStream/Types.hs deleted file mode 100644 index 29cf961781..0000000000 --- a/src/Streamly/Internal/Data/Stream/IsStream/Types.hs +++ /dev/null @@ -1,74 +0,0 @@ --- | --- Module : Streamly.Internal.Data.Stream.IsStream --- Copyright : (c) 2017 Composewell Technologies --- License : BSD-3-Clause --- Maintainer : streamly@composewell.com --- Stability : experimental --- Portability : GHC --- --- | This module contains different stream types and combinators to --- interconvert between them. - -module Streamly.Internal.Data.Stream.IsStream.Types - ( - -- * Stream Types - -- ** Serial Streams - SerialT - , Serial - , WSerialT - , WSerial - - -- ** Speculative Streams - , AheadT - , Ahead - - -- ** Asynchronous Streams - , AsyncT - , Async - , WAsyncT - , WAsync - - -- ** Parallel Streams - -- | Ahead, Async and WAsync schedule actions concurrently on demand. - -- Unlike those 'Parallel' streams schedule all actions concurrently - -- upfront. - , ParallelT - , Parallel - , mkAsync - - -- ** Zipping Streams - , ZipSerialM - , ZipSerial - , ZipAsyncM - , ZipAsync - - -- * Stream Type Adapters - , IsStream () - - , fromSerial - , fromWSerial - , fromAsync - , fromAhead - , fromWAsync - , fromParallel - , fromZipSerial - , fromZipAsync - , adapt - - -- * Type Synonyms - , MonadAsync - ) -where - -import Streamly.Internal.Data.Stream.Ahead (AheadT, Ahead, fromAhead) -import Streamly.Internal.Data.Stream.Async - ( AsyncT, Async, WAsyncT, WAsync, mkAsync, fromAsync - , fromWAsync) -import Streamly.Internal.Data.Stream.Parallel (ParallelT, Parallel, fromParallel) -import Streamly.Internal.Data.Stream.Serial - ( SerialT, WSerialT, Serial, WSerial, fromSerial - , fromWSerial) -import Streamly.Internal.Data.Stream.StreamK (IsStream(), adapt) -import Streamly.Internal.Data.Stream.Zip - (ZipSerialM, ZipSerial, ZipAsyncM, ZipAsync, fromZipSerial, fromZipAsync) -import Streamly.Internal.Control.Concurrent (MonadAsync) diff --git a/src/Streamly/Internal/Data/Stream/Parallel.hs b/src/Streamly/Internal/Data/Stream/Parallel.hs index b4019446ca..61dd5c2d48 100644 --- a/src/Streamly/Internal/Data/Stream/Parallel.hs +++ b/src/Streamly/Internal/Data/Stream/Parallel.hs @@ -1,7 +1,5 @@ {-# LANGUAGE UndecidableInstances #-} -#include "inline.hs" - -- | -- Module : Streamly.Internal.Data.Stream.Parallel -- Copyright : (c) 2017 Composewell Technologies @@ -25,24 +23,22 @@ module Streamly.Internal.Data.Stream.Parallel ( -- * Parallel Stream Type - ParallelT + ParallelT(..) , Parallel - , fromParallel + , consM -- * Merge Concurrently - , parallel - , parallelFst - , parallelMin + , parallelK + , parallelFstK + , parallelMinK -- * Evaluate Concurrently - , mkParallel , mkParallelD , mkParallelK -- * Tap Concurrently - , tapAsync + , tapAsyncK , tapAsyncF - , distributeAsync_ -- * Callbacks , newCallbackStream @@ -70,11 +66,10 @@ import qualified Data.Set as Set import Streamly.Internal.Control.Concurrent (MonadAsync) import Streamly.Internal.Data.Fold.Type (Fold) +import Streamly.Internal.Data.Stream.Serial (SerialT(..)) import Streamly.Internal.Data.Stream.StreamD.Type (Step(..)) -import Streamly.Internal.Data.Stream.StreamK - (IsStream(..), Stream, mkStream, foldStream, foldStreamShared, adapt) +import Streamly.Internal.Data.Stream.StreamK.Type (Stream) -import qualified Streamly.Internal.Data.Stream.StreamK as K (withLocal) import qualified Streamly.Internal.Data.Stream.StreamK.Type as K import qualified Streamly.Internal.Data.Stream.StreamD.Type as D import qualified Streamly.Internal.Data.Stream.SVar.Generate as SVar @@ -82,6 +77,7 @@ import qualified Streamly.Internal.Data.Stream.SVar.Eliminate as SVar import Streamly.Internal.Data.SVar +#include "inline.hs" #include "Instances.hs" -- @@ -116,7 +112,7 @@ runOne st m0 winfo = go m = do liftIO $ decrementBufferLimit sv - foldStreamShared st yieldk single stop m + K.foldStreamShared st yieldk single stop m sv = fromJust $ streamVar st @@ -139,7 +135,7 @@ runOneLimited st m0 winfo = go m0 if yieldLimitOk then do liftIO $ decrementBufferLimit sv - foldStreamShared st yieldk single stop m + K.foldStreamShared st yieldk single stop m else do liftIO $ cleanupSVarFromWorker sv liftIO $ sendStop sv winfo @@ -165,23 +161,23 @@ runOneLimited st m0 winfo = go m0 -- remember the past state. {-# NOINLINE forkSVarPar #-} -forkSVarPar :: (IsStream t, MonadAsync m) - => SVarStopStyle -> t m a -> t m a -> t m a -forkSVarPar ss m r = mkStream $ \st yld sng stp -> do +forkSVarPar :: MonadAsync m + => SVarStopStyle -> Stream m a -> Stream m a -> Stream m a +forkSVarPar ss m r = K.mkStream $ \st yld sng stp -> do sv <- newParallelVar ss st - pushWorkerPar sv (runOne st{streamVar = Just sv} $ toStream m) + pushWorkerPar sv (runOne st{streamVar = Just sv} m) case ss of StopBy -> liftIO $ do set <- readIORef (workerThreads sv) writeIORef (svarStopBy sv) $ Set.elemAt 0 set _ -> return () - pushWorkerPar sv (runOne st{streamVar = Just sv} $ toStream r) - foldStream st yld sng stp (SVar.fromSVar sv) + pushWorkerPar sv (runOne st{streamVar = Just sv} r) + K.foldStream st yld sng stp $ getSerialT (SVar.fromSVar sv) {-# INLINE joinStreamVarPar #-} -joinStreamVarPar :: (IsStream t, MonadAsync m) - => SVarStyle -> SVarStopStyle -> t m a -> t m a -> t m a -joinStreamVarPar style ss m1 m2 = mkStream $ \st yld sng stp -> +joinStreamVarPar :: MonadAsync m + => SVarStyle -> SVarStopStyle -> Stream m a -> Stream m a -> Stream m a +joinStreamVarPar style ss m1 m2 = K.mkStream $ \st yld sng stp -> case streamVar st of Just sv | svarStyle sv == style && svarStopStyle sv == ss -> do -- Here, WE ARE IN THE WORKER/PRODUCER THREAD, we know that because @@ -217,56 +213,28 @@ joinStreamVarPar style ss m1 m2 = mkStream $ \st yld sng stp -> -- worker spawned by that worker would fork, thus creating layer -- over layer of workers and a chain of threads leading to a very -- inefficient execution. - pushWorkerPar sv (runOne st $ toStream m1) - foldStreamShared st yld sng stp m2 + pushWorkerPar sv (runOne st m1) + K.foldStreamShared st yld sng stp m2 _ -> -- Here WE ARE IN THE CONSUMER THREAD, we create a new SVar, fork -- worker threads to execute m1 and m2 and this thread starts -- pulling the stream from the SVar. - foldStreamShared st yld sng stp (forkSVarPar ss m1 m2) + K.foldStreamShared st yld sng stp (forkSVarPar ss m1 m2) ------------------------------------------------------------------------------- -- User facing APIs ------------------------------------------------------------------------------- +{-# INLINE parallelK #-} +parallelK :: MonadAsync m => Stream m a -> Stream m a -> Stream m a +parallelK = joinStreamVarPar ParallelVar StopNone + -- | XXX we can implement it more efficienty by directly implementing instead -- of combining streams using parallel. -{-# INLINE consMParallel #-} -{-# SPECIALIZE consMParallel :: IO a -> ParallelT IO a -> ParallelT IO a #-} -consMParallel :: MonadAsync m => m a -> ParallelT m a -> ParallelT m a -consMParallel m r = fromStream $ K.fromEffect m `parallel` toStream r - -infixr 6 `parallel` - --- | Like 'Streamly.Prelude.async' except that the execution is much more --- strict. There is no limit on the number of threads. While --- 'Streamly.Prelude.async' may not schedule a stream if there is no demand --- from the consumer, 'parallel' always evaluates both the streams immediately. --- The only limit that applies to 'parallel' is 'Streamly.Prelude.maxBuffer'. --- Evaluation may block if the output buffer becomes full. --- --- >>> import Streamly.Prelude (parallel) --- >>> stream = Stream.fromEffect (delay 2) `parallel` Stream.fromEffect (delay 1) --- >>> Stream.toList stream -- IO [Int] --- 1 sec --- 2 sec --- [1,2] --- --- 'parallel' guarantees that all the streams are scheduled for execution --- immediately, therefore, we could use things like starting timers inside the --- streams and relying on the fact that all timers were started at the same --- time. --- --- Unlike 'async' this operation cannot be used to fold an infinite lazy --- container of streams, because it schedules all the streams strictly --- concurrently. --- --- /Since: 0.2.0 ("Streamly")/ --- --- @since 0.8.0 -{-# INLINE parallel #-} -parallel :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a -parallel = joinStreamVarPar ParallelVar StopNone +{-# INLINE consM #-} +{-# SPECIALIZE consM :: IO a -> ParallelT IO a -> ParallelT IO a #-} +consM :: MonadAsync m => m a -> ParallelT m a -> ParallelT m a +consM m (ParallelT r) = ParallelT $ parallelK (K.fromEffect m) r -- This is a co-parallel like combinator for streams, where first stream is the -- main stream and the rest are just supporting it, when the first ends @@ -275,9 +243,9 @@ parallel = joinStreamVarPar ParallelVar StopNone -- | Like `parallel` but stops the output as soon as the first stream stops. -- -- /Pre-release/ -{-# INLINE parallelFst #-} -parallelFst :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a -parallelFst = joinStreamVarPar ParallelVar StopBy +{-# INLINE parallelFstK #-} +parallelFstK :: MonadAsync m => Stream m a -> Stream m a -> Stream m a +parallelFstK = joinStreamVarPar ParallelVar StopBy -- This is a race like combinator for streams. -- @@ -285,9 +253,9 @@ parallelFst = joinStreamVarPar ParallelVar StopBy -- stops. -- -- /Pre-release/ -{-# INLINE parallelMin #-} -parallelMin :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a -parallelMin = joinStreamVarPar ParallelVar StopAny +{-# INLINE parallelMinK #-} +parallelMinK :: MonadAsync m => Stream m a -> Stream m a -> Stream m a +parallelMinK = joinStreamVarPar ParallelVar StopAny ------------------------------------------------------------------------------ -- Convert a stream to parallel @@ -297,16 +265,12 @@ parallelMin = joinStreamVarPar ParallelVar StopAny -- -- /Pre-release/ -- -mkParallelK :: (IsStream t, MonadAsync m) => t m a -> t m a -mkParallelK m = mkStream $ \st yld sng stp -> do +mkParallelK :: MonadAsync m => Stream m a -> Stream m a +mkParallelK m = K.mkStream $ \st yld sng stp -> do sv <- newParallelVar StopNone (adaptState st) -- pushWorkerPar sv (runOne st{streamVar = Just sv} $ toStream m) - SVar.toSVarParallel st sv $ D.toStreamD m - foldStream st yld sng stp $ SVar.fromSVar sv - -------------------------------------------------------------------------------- --- Concurrent application and fold -------------------------------------------------------------------------------- + SVar.toSVarParallel st sv $ D.fromStreamK m + K.foldStream st yld sng stp $ getSerialT $ SVar.fromSVar sv -- | Same as 'mkParallel' but for StreamD stream. -- @@ -328,25 +292,6 @@ mkParallelD m = D.Stream step Nothing Skip s -> Skip (Just $ D.Stream step1 s) Stop -> Stop --- Compare with mkAsync. mkAsync uses an Async style SVar whereas this uses a --- parallel style SVar for evaluation. Currently, parallel style cannot use --- rate control whereas Async style can use rate control. In async style SVar --- the worker thread terminates when the buffer is full whereas in Parallel --- style it blocks. --- --- | Make the stream producer and consumer run concurrently by introducing a --- buffer between them. The producer thread evaluates the input stream until --- the buffer fills, it blocks if the buffer is full until there is space in --- the buffer. The consumer consumes the stream lazily from the buffer. --- --- @mkParallel = D.fromStreamD . mkParallelD . D.toStreamD@ --- --- /Pre-release/ --- -{-# INLINE_NORMAL mkParallel #-} -mkParallel :: (K.IsStream t, MonadAsync m) => t m a -> t m a -mkParallel = D.fromStreamD . mkParallelD . D.toStreamD - ------------------------------------------------------------------------------- -- Concurrent tap ------------------------------------------------------------------------------- @@ -392,11 +337,12 @@ mkParallel = D.fromStreamD . mkParallelD . D.toStreamD -- Compare with 'tap'. -- -- /Pre-release/ -{-# INLINE tapAsync #-} -tapAsync :: (IsStream t, MonadAsync m) => (t m a -> m b) -> t m a -> t m a -tapAsync f m = mkStream $ \st yld sng stp -> do - sv <- SVar.newFoldSVar st f - foldStreamShared st yld sng stp (SVar.teeToSVar sv m) +{-# INLINE tapAsyncK #-} +tapAsyncK :: MonadAsync m => (Stream m a -> m b) -> Stream m a -> Stream m a +tapAsyncK f m = K.mkStream $ \st yld sng stp -> do + sv <- SVar.newFoldSVar st (f . getSerialT) + K.foldStreamShared st yld sng stp + $ getSerialT (SVar.teeToSVar sv $ SerialT m) data TapState fs st a = TapInit | Tapping !fs st | TapDone st @@ -450,29 +396,6 @@ tapAsyncF f (D.Stream step1 state1) = D.Stream step TapInit Skip s -> Skip (TapDone s) Stop -> Stop --- | Concurrently distribute a stream to a collection of fold functions, --- discarding the outputs of the folds. --- --- @ --- > Stream.drain $ Stream.distributeAsync_ [Stream.mapM_ print, Stream.mapM_ print] (Stream.enumerateFromTo 1 2) --- 1 --- 2 --- 1 --- 2 --- --- @ --- --- @ --- distributeAsync_ = flip (foldr tapAsync) --- @ --- --- /Pre-release/ --- -{-# INLINE distributeAsync_ #-} -distributeAsync_ :: (Foldable f, IsStream t, MonadAsync m) - => f (t m a -> m b) -> t m a -> t m a -distributeAsync_ = flip (foldr tapAsync) - ------------------------------------------------------------------------------ -- ParallelT ------------------------------------------------------------------------------ @@ -504,44 +427,24 @@ newtype ParallelT m a = ParallelT {getParallelT :: Stream m a} -- @since 0.8.0 type Parallel = ParallelT IO --- | Fix the type of a polymorphic stream as 'ParallelT'. --- --- /Since: 0.1.0 ("Streamly")/ --- --- @since 0.8.0 -fromParallel :: IsStream t => ParallelT m a -> t m a -fromParallel = adapt - -instance IsStream ParallelT where - toStream = getParallelT - fromStream = ParallelT - - {-# INLINE consM #-} - {-# SPECIALIZE consM :: IO a -> ParallelT IO a -> ParallelT IO a #-} - consM = consMParallel - - {-# INLINE (|:) #-} - {-# SPECIALIZE (|:) :: IO a -> ParallelT IO a -> ParallelT IO a #-} - (|:) = consM - ------------------------------------------------------------------------------ -- Semigroup ------------------------------------------------------------------------------ -{-# INLINE mappendParallel #-} -{-# SPECIALIZE mappendParallel :: ParallelT IO a -> ParallelT IO a -> ParallelT IO a #-} -mappendParallel :: MonadAsync m => ParallelT m a -> ParallelT m a -> ParallelT m a -mappendParallel m1 m2 = fromStream $ parallel (toStream m1) (toStream m2) +{-# INLINE append #-} +{-# SPECIALIZE append :: ParallelT IO a -> ParallelT IO a -> ParallelT IO a #-} +append :: MonadAsync m => ParallelT m a -> ParallelT m a -> ParallelT m a +append (ParallelT m1) (ParallelT m2) = ParallelT $ parallelK m1 m2 instance MonadAsync m => Semigroup (ParallelT m a) where - (<>) = mappendParallel + (<>) = append ------------------------------------------------------------------------------ -- Monoid ------------------------------------------------------------------------------ instance MonadAsync m => Monoid (ParallelT m a) where - mempty = K.nil + mempty = ParallelT K.nil mappend = (<>) ------------------------------------------------------------------------------ @@ -549,15 +452,18 @@ instance MonadAsync m => Monoid (ParallelT m a) where ------------------------------------------------------------------------------ {-# INLINE apParallel #-} -{-# SPECIALIZE apParallel :: ParallelT IO (a -> b) -> ParallelT IO a -> ParallelT IO b #-} -apParallel :: MonadAsync m => ParallelT m (a -> b) -> ParallelT m a -> ParallelT m b +{-# SPECIALIZE apParallel :: + ParallelT IO (a -> b) -> ParallelT IO a -> ParallelT IO b #-} +apParallel :: MonadAsync m => + ParallelT m (a -> b) -> ParallelT m a -> ParallelT m b apParallel (ParallelT m1) (ParallelT m2) = - let f x1 = K.concatMapBy parallel (pure . x1) m2 - in ParallelT $ K.concatMapBy parallel f m1 + let f x1 = K.concatMapWith parallelK (pure . x1) m2 + in ParallelT $ K.concatMapWith parallelK f m1 instance (Monad m, MonadAsync m) => Applicative (ParallelT m) where {-# INLINE pure #-} pure = ParallelT . K.fromPure + {-# INLINE (<*>) #-} (<*>) = apParallel @@ -565,14 +471,17 @@ instance (Monad m, MonadAsync m) => Applicative (ParallelT m) where -- Monad ------------------------------------------------------------------------------ -{-# INLINE bindParallel #-} -{-# SPECIALIZE bindParallel :: ParallelT IO a -> (a -> ParallelT IO b) -> ParallelT IO b #-} -bindParallel :: MonadAsync m => ParallelT m a -> (a -> ParallelT m b) -> ParallelT m b -bindParallel m f = fromStream $ K.bindWith parallel (K.adapt m) (K.adapt . f) +{-# INLINE bind #-} +{-# SPECIALIZE bind :: + ParallelT IO a -> (a -> ParallelT IO b) -> ParallelT IO b #-} +bind :: MonadAsync m => ParallelT m a -> (a -> ParallelT m b) -> ParallelT m b +bind (ParallelT m) f = ParallelT $ K.bindWith parallelK m (getParallelT . f) instance MonadAsync m => Monad (ParallelT m) where return = pure - (>>=) = bindParallel + + {-# INLINE (>>=) #-} + (>>=) = bind ------------------------------------------------------------------------------ -- Other instances @@ -594,7 +503,7 @@ MONAD_COMMON_INSTANCES(ParallelT, MONADPARALLEL) -- /Pre-release/ -- {-# INLINE_NORMAL newCallbackStream #-} -newCallbackStream :: (K.IsStream t, MonadAsync m) => m (a -> m (), t m a) +newCallbackStream :: MonadAsync m => m (a -> m (), Stream m a) newCallbackStream = do sv <- newParallelVar StopNone defState @@ -606,4 +515,4 @@ newCallbackStream = do let callback a = liftIO $ void $ send sv (ChildYield a) -- XXX we can return an SVar and then the consumer can unfold from the -- SVar? - return (callback, D.fromStreamD (SVar.fromSVarD sv)) + return (callback, D.toStreamK (SVar.fromSVarD sv)) diff --git a/src/Streamly/Internal/Data/Stream/Prelude.hs b/src/Streamly/Internal/Data/Stream/Prelude.hs index 7407345393..8ce5ad007a 100644 --- a/src/Streamly/Internal/Data/Stream/Prelude.hs +++ b/src/Streamly/Internal/Data/Stream/Prelude.hs @@ -1,25 +1,22 @@ +{-# LANGUAGE UndecidableInstances #-} {-# OPTIONS_GHC -Wno-orphans #-} -#include "inline.hs" - -- | -- Module : Streamly.Internal.Data.Stream.Prelude -- Copyright : (c) 2017 Composewell Technologies --- --- License : BSD3 +-- License : BSD-3-Clause -- Maintainer : streamly@composewell.com -- Stability : experimental -- Portability : GHC -- +-- Low level functions using StreamK as the intermediate stream type. These +-- functions are used in SerialT/AsyncT/AheadT/ParallelT stream modules to +-- implement their instances.. -- module Streamly.Internal.Data.Stream.Prelude ( - -- * Stream Conversion - fromStreamS - , toStreamS - -- * Running Effects - , drain + drain -- * Conversion operations , fromList @@ -35,318 +32,26 @@ module Streamly.Internal.Data.Stream.Prelude , foldl' , fold - -- Lazy left folds are useful only for reversing the stream - , foldlS - , foldlT - - , scanlx' - , scanlMx' - , postscanlx' - , postscanlMx' - , postscanOnce - , scanOnce - -- * Zip style operations , eqBy , cmpBy - - -- * Foldable instance - , minimum - , maximum - - -- * Nesting - , K.concatMapBy - , K.concatMap - - -- * Fold Utilities - , concatFoldableWith - , concatMapFoldableWith - , concatForFoldableWith ) where -import Control.Monad.Trans.Class (MonadTrans(..)) -import Prelude hiding (foldr, minimum, maximum) -import qualified Prelude +#include "inline.hs" import Streamly.Internal.Data.Fold.Type (Fold (..)) #ifdef USE_STREAMK_ONLY import qualified Streamly.Internal.Data.Stream.StreamK as S +import qualified Streamly.Internal.Data.Stream.StreamK.Type as S #else -import qualified Streamly.Internal.Data.Stream.StreamD as S +import qualified Streamly.Internal.Data.Stream.StreamD.Type as S #endif -import Streamly.Internal.Data.Stream.StreamK (IsStream(..)) -import qualified Streamly.Internal.Data.Stream.StreamK as K -import qualified Streamly.Internal.Data.Stream.StreamD as D - ------------------------------------------------------------------------------- --- Conversion to and from direct style stream ------------------------------------------------------------------------------- - --- These definitions are dependent on what is imported as S -{-# INLINE fromStreamS #-} -fromStreamS :: (IsStream t, Monad m) => S.Stream m a -> t m a -fromStreamS = fromStream . S.toStreamK - -{-# INLINE toStreamS #-} -toStreamS :: (IsStream t, Monad m) => t m a -> S.Stream m a -toStreamS = S.fromStreamK . toStream - ------------------------------------------------------------------------------- --- Conversions ------------------------------------------------------------------------------- - -{-# INLINE_EARLY drain #-} -drain :: (IsStream t, Monad m) => t m a -> m () -drain m = D.drain $ D.fromStreamK (toStream m) -{-# RULES "drain fallback to CPS" [1] - forall a. D.drain (D.fromStreamK a) = K.drain a #-} - ------------------------------------------------------------------------------- --- Conversions ------------------------------------------------------------------------------- - --- | --- @ --- fromList = 'Prelude.foldr' 'K.cons' 'K.nil' --- @ --- --- Construct a stream from a list of pure values. This is more efficient than --- 'K.fromFoldable' for serial streams. --- --- @since 0.4.0 -{-# INLINE_EARLY fromList #-} -fromList :: (Monad m, IsStream t) => [a] -> t m a -fromList = fromStreamS . S.fromList -{-# RULES "fromList fallback to StreamK" [1] - forall a. S.toStreamK (S.fromList a) = K.fromFoldable a #-} - --- | Convert a stream into a list in the underlying monad. --- --- @since 0.1.0 -{-# INLINE toList #-} -toList :: (Monad m, IsStream t) => t m a -> m [a] -toList m = S.toList $ toStreamS m - ------------------------------------------------------------------------------- --- Folds ------------------------------------------------------------------------------- - -{-# INLINE foldrM #-} -foldrM :: (Monad m, IsStream t) => (a -> m b -> m b) -> m b -> t m a -> m b -foldrM step acc m = S.foldrM step acc $ toStreamS m - -{-# INLINE foldrMx #-} -foldrMx :: (Monad m, IsStream t) - => (a -> m x -> m x) -> m x -> (m x -> m b) -> t m a -> m b -foldrMx step final project m = D.foldrMx step final project $ D.toStreamD m - -{-# INLINE foldr #-} -foldr :: (Monad m, IsStream t) => (a -> b -> b) -> b -> t m a -> m b -foldr f z = foldrM (\a b -> f a <$> b) (return z) - --- | Like 'foldlx'', but with a monadic step function. --- --- @since 0.7.0 -{-# INLINE foldlMx' #-} -foldlMx' :: (IsStream t, Monad m) - => (x -> a -> m x) -> m x -> (x -> m b) -> t m a -> m b -foldlMx' step begin done m = S.foldlMx' step begin done $ toStreamS m - --- | Strict left fold with an extraction function. Like the standard strict --- left fold, but applies a user supplied extraction function (the third --- argument) to the folded value at the end. This is designed to work with the --- @foldl@ library. The suffix @x@ is a mnemonic for extraction. --- --- @since 0.7.0 -{-# INLINE foldlx' #-} -foldlx' :: (IsStream t, Monad m) - => (x -> a -> x) -> x -> (x -> b) -> t m a -> m b -foldlx' step begin done m = S.foldlx' step begin done $ toStreamS m - --- | Strict left associative fold. --- --- @since 0.2.0 -{-# INLINE foldl' #-} -foldl' :: (Monad m, IsStream t) => (b -> a -> b) -> b -> t m a -> m b -foldl' step begin m = S.foldl' step begin $ toStreamS m - -{-# INLINE foldlS #-} -foldlS :: IsStream t => (t m b -> a -> t m b) -> t m b -> t m a -> t m b -foldlS = K.foldlS - --- | Lazy left fold to a transformer monad. --- --- For example, to reverse a stream: --- --- > S.toList $ S.foldlT (flip S.cons) S.nil $ (S.fromList [1..5] :: SerialT IO Int) --- -{-# INLINE foldlT #-} -foldlT :: (Monad m, IsStream t, Monad (s m), MonadTrans s) - => (s m b -> a -> s m b) -> s m b -> t m a -> s m b -foldlT f z s = S.foldlT f z (toStreamS s) - -{-# INLINE fold #-} -fold :: (Monad m, IsStream t) => Fold m a b -> t m a -> m b -fold fld m = S.fold fld $ toStreamS m +import qualified Streamly.Internal.Data.Stream.StreamK.Type as K +import qualified Streamly.Internal.Data.Stream.StreamD.Type as D ------------------------------------------------------------------------------- --- Scans ------------------------------------------------------------------------------- +import Prelude hiding (foldr, repeat) --- postscanlM' followed by mapM -{-# INLINE postscanlMx' #-} -postscanlMx' :: (IsStream t, Monad m) - => (x -> a -> m x) -> m x -> (x -> m b) -> t m a -> t m b -postscanlMx' step begin done m = - D.fromStreamD $ D.postscanlMx' step begin done $ D.toStreamD m - --- postscanl' followed by map -{-# INLINE postscanlx' #-} -postscanlx' :: (IsStream t, Monad m) - => (x -> a -> x) -> x -> (x -> b) -> t m a -> t m b -postscanlx' step begin done m = - D.fromStreamD $ D.postscanlx' step begin done $ D.toStreamD m - --- scanlM' followed by mapM --- -{-# INLINE scanlMx' #-} -scanlMx' :: (IsStream t, Monad m) - => (x -> a -> m x) -> m x -> (x -> m b) -> t m a -> t m b -scanlMx' step begin done m = - D.fromStreamD $ D.scanlMx' step begin done $ D.toStreamD m - -{-# INLINE_NORMAL postscanOnce #-} -postscanOnce :: (IsStream t, Monad m) - => Fold m a b -> t m a -> t m b -postscanOnce fld m = - D.fromStreamD $ D.postscanOnce fld $ D.toStreamD m - -{-# INLINE scanOnce #-} -scanOnce :: (IsStream t, Monad m) - => Fold m a b -> t m a -> t m b -scanOnce fld m = D.fromStreamD $ D.scanOnce fld $ D.toStreamD m - --- scanl followed by map --- --- | Strict left scan with an extraction function. Like 'scanl'', but applies a --- user supplied extraction function (the third argument) at each step. This is --- designed to work with the @foldl@ library. The suffix @x@ is a mnemonic for --- extraction. --- --- @since 0.7.0 -{-# INLINE scanlx' #-} -scanlx' :: (IsStream t, Monad m) - => (x -> a -> x) -> x -> (x -> b) -> t m a -> t m b -scanlx' step begin done m = - fromStreamS $ S.scanlx' step begin done $ toStreamS m - ------------------------------------------------------------------------------- --- Comparison ------------------------------------------------------------------------------- - --- | Compare two streams for equality --- --- @since 0.5.3 -{-# INLINE eqBy #-} -eqBy :: (IsStream t, Monad m) => (a -> b -> Bool) -> t m a -> t m b -> m Bool -eqBy f m1 m2 = D.eqBy f (D.toStreamD m1) (D.toStreamD m2) - --- | Compare two streams --- --- @since 0.5.3 -{-# INLINE cmpBy #-} -cmpBy - :: (IsStream t, Monad m) - => (a -> b -> Ordering) -> t m a -> t m b -> m Ordering -cmpBy f m1 m2 = D.cmpBy f (D.toStreamD m1) (D.toStreamD m2) - -{-# INLINE minimum #-} -minimum :: (IsStream t, Monad m, Ord a) => t m a -> m (Maybe a) -minimum m = S.minimum (toStreamS m) - -{-# INLINE maximum #-} -maximum :: (IsStream t, Monad m, Ord a) => t m a -> m (Maybe a) -maximum m = S.maximum (toStreamS m) - ------------------------------------------------------------------------------- --- Fold Utilities ------------------------------------------------------------------------------- - -{- --- XXX do we have facilities in Foldable to fold any Foldable in this manner? --- --- | Perform a pair wise bottom up hierarchical fold of elements in the --- container using the given function as the merge function. --- --- This will perform a balanced merge sort if the merge function is --- 'mergeBy compare'. --- --- @since 0.7.0 -{-# INLINABLE foldbWith #-} -foldbWith :: IsStream t - => (t m a -> t m a -> t m a) -> SerialT Identity (t m a) -> t m a -foldbWith f = K.foldb f K.nil --} - - --- | A variant of 'Data.Foldable.fold' that allows you to fold a 'Foldable' --- container of streams using the specified stream sum operation. --- --- @concatFoldableWith 'async' $ map return [1..3]@ --- --- Equivalent to: --- --- @ --- concatFoldableWith f = Prelude.foldr f S.nil --- concatFoldableWith f = S.concatMapFoldableWith f id --- @ --- --- /Since: 0.8.0 (Renamed foldWith to concatFoldableWith)/ --- --- /Since: 0.1.0 ("Streamly")/ -{-# INLINABLE concatFoldableWith #-} -concatFoldableWith :: (IsStream t, Foldable f) - => (t m a -> t m a -> t m a) -> f (t m a) -> t m a -concatFoldableWith f = Prelude.foldr f K.nil - --- | A variant of 'foldMap' that allows you to map a monadic streaming action --- on a 'Foldable' container and then fold it using the specified stream merge --- operation. --- --- @concatMapFoldableWith 'async' return [1..3]@ --- --- Equivalent to: --- --- @ --- concatMapFoldableWith f g = Prelude.foldr (f . g) S.nil --- concatMapFoldableWith f g xs = S.concatMapWith f g (S.fromFoldable xs) --- @ --- --- /Since: 0.8.0 (Renamed foldMapWith to concatMapFoldableWith)/ --- --- /Since: 0.1.0 ("Streamly")/ -{-# INLINABLE concatMapFoldableWith #-} -concatMapFoldableWith :: (IsStream t, Foldable f) - => (t m b -> t m b -> t m b) -> (a -> t m b) -> f a -> t m b -concatMapFoldableWith f g = Prelude.foldr (f . g) K.nil - --- | Like 'concatMapFoldableWith' but with the last two arguments reversed i.e. the --- monadic streaming function is the last argument. --- --- Equivalent to: --- --- @ --- concatForFoldableWith f xs g = Prelude.foldr (f . g) S.nil xs --- concatForFoldableWith = flip S.concatMapFoldableWith --- @ --- --- /Since: 0.8.0 (Renamed forEachWith to concatForFoldableWith)/ --- --- /Since: 0.1.0 ("Streamly")/ -{-# INLINABLE concatForFoldableWith #-} -concatForFoldableWith :: (IsStream t, Foldable f) - => (t m b -> t m b -> t m b) -> f a -> (a -> t m b) -> t m b -concatForFoldableWith f xs g = Prelude.foldr (f . g) K.nil xs +#include "PreludeCommon.hs" diff --git a/src/Streamly/Internal/Data/Stream/PreludeCommon.hs b/src/Streamly/Internal/Data/Stream/PreludeCommon.hs new file mode 100644 index 0000000000..4e109ea49d --- /dev/null +++ b/src/Streamly/Internal/Data/Stream/PreludeCommon.hs @@ -0,0 +1,139 @@ +------------------------------------------------------------------------------ +-- Conversion to and from direct style stream +------------------------------------------------------------------------------ + +#ifdef USE_IS_STREAM +#define IS_STREAM IsStream t, +#define FROM_STREAM fromStream +#define TO_STREAM toStream +#define STREAM t +#else +#define IS_STREAM +#define FROM_STREAM id +#define TO_STREAM id +#define STREAM K.Stream +#endif + +-- These definitions are dependent on what is imported as S +{-# INLINE fromStreamS #-} +fromStreamS :: (IS_STREAM Monad m) => S.Stream m a -> STREAM m a +fromStreamS = FROM_STREAM . S.toStreamK + +{-# INLINE toStreamS #-} +toStreamS :: (IS_STREAM Monad m) => STREAM m a -> S.Stream m a +toStreamS = S.fromStreamK . TO_STREAM + +{-# INLINE toStreamD #-} +toStreamD :: (IS_STREAM Monad m) => STREAM m a -> D.Stream m a +toStreamD = D.fromStreamK . TO_STREAM + +------------------------------------------------------------------------------ +-- Conversions +------------------------------------------------------------------------------ + +{-# INLINE_EARLY drain #-} +drain :: (IS_STREAM Monad m) => STREAM m a -> m () +drain m = D.drain $ D.fromStreamK (TO_STREAM m) +{-# RULES "drain fallback to CPS" [1] + forall a. D.drain (D.fromStreamK a) = K.drain a #-} + +------------------------------------------------------------------------------ +-- Conversions +------------------------------------------------------------------------------ + +-- | +-- @ +-- fromList = 'Prelude.foldr' 'K.cons' 'K.nil' +-- @ +-- +-- Construct a stream from a list of pure values. This is more efficient than +-- 'K.fromFoldable' for serial streams. +-- +-- @since 0.4.0 +{-# INLINE_EARLY fromList #-} +#ifdef USE_IS_STREAM +fromList :: (Monad m, IsStream t) => [a] -> STREAM m a +#else +fromList :: Monad m => [a] -> STREAM m a +#endif +fromList = fromStreamS . S.fromList +{-# RULES "fromList fallback to StreamK" [1] + forall a. S.toStreamK (S.fromList a) = K.fromFoldable a #-} + +-- | Convert a stream into a list in the underlying monad. +-- +-- @since 0.1.0 +{-# INLINE toList #-} +toList :: (IS_STREAM Monad m) => STREAM m a -> m [a] +toList m = S.toList $ toStreamS m + +------------------------------------------------------------------------------ +-- Folds +------------------------------------------------------------------------------ + +{-# INLINE foldrM #-} +foldrM :: (IS_STREAM Monad m) => (a -> m b -> m b) -> m b -> STREAM m a -> m b +foldrM step acc m = S.foldrM step acc $ toStreamS m + +{-# INLINE foldrMx #-} +foldrMx :: (IS_STREAM Monad m) + => (a -> m x -> m x) -> m x -> (m x -> m b) -> STREAM m a -> m b +foldrMx step final project m = D.foldrMx step final project $ toStreamD m + +{-# INLINE foldr #-} +foldr :: (IS_STREAM Monad m) => (a -> b -> b) -> b -> STREAM m a -> m b +foldr f z = foldrM (\a b -> f a <$> b) (return z) + +-- | Like 'foldlx'', but with a monadic step function. +-- +-- @since 0.7.0 +{-# INLINE foldlMx' #-} +foldlMx' :: + (IS_STREAM Monad m) + => (x -> a -> m x) -> m x -> (x -> m b) -> STREAM m a -> m b +foldlMx' step begin done m = S.foldlMx' step begin done $ toStreamS m + +-- | Strict left fold with an extraction function. Like the standard strict +-- left fold, but applies a user supplied extraction function (the third +-- argument) to the folded value at the end. This is designed to work with the +-- @foldl@ library. The suffix @x@ is a mnemonic for extraction. +-- +-- @since 0.7.0 +{-# INLINE foldlx' #-} +foldlx' :: + (IS_STREAM Monad m) => (x -> a -> x) -> x -> (x -> b) -> STREAM m a -> m b +foldlx' step begin done m = S.foldlx' step begin done $ toStreamS m + +-- | Strict left associative fold. +-- +-- @since 0.2.0 +{-# INLINE foldl' #-} +foldl' :: + (IS_STREAM Monad m) => (b -> a -> b) -> b -> STREAM m a -> m b +foldl' step begin m = S.foldl' step begin $ toStreamS m + + +{-# INLINE fold #-} +fold :: (IS_STREAM Monad m) => Fold m a b -> STREAM m a -> m b +fold fld m = S.fold fld $ toStreamS m + +------------------------------------------------------------------------------ +-- Comparison +------------------------------------------------------------------------------ + +-- | Compare two streams for equality +-- +-- @since 0.5.3 +{-# INLINE eqBy #-} +eqBy :: (IS_STREAM Monad m) => + (a -> b -> Bool) -> STREAM m a -> STREAM m b -> m Bool +eqBy f m1 m2 = D.eqBy f (toStreamD m1) (toStreamD m2) + +-- | Compare two streams +-- +-- @since 0.5.3 +{-# INLINE cmpBy #-} +cmpBy + :: (IS_STREAM Monad m) + => (a -> b -> Ordering) -> STREAM m a -> STREAM m b -> m Ordering +cmpBy f m1 m2 = D.cmpBy f (toStreamD m1) (toStreamD m2) diff --git a/src/Streamly/Internal/Data/Stream/SVar/Eliminate.hs b/src/Streamly/Internal/Data/Stream/SVar/Eliminate.hs index 8cdc3c63dd..8bf4f7e883 100644 --- a/src/Streamly/Internal/Data/Stream/SVar/Eliminate.hs +++ b/src/Streamly/Internal/Data/Stream/SVar/Eliminate.hs @@ -35,6 +35,7 @@ import Streamly.Internal.Control.Concurrent (MonadAsync, doFork) import Streamly.Internal.Data.Atomics (atomicModifyIORefCAS_) import Streamly.Internal.Data.Fold.SVar (write, writeLimited) import Streamly.Internal.Data.Fold.Type (Fold(..)) +import Streamly.Internal.Data.Stream.Serial (SerialT(..)) import Streamly.Internal.Data.Time.Clock (Clock(Monotonic), getTime) import qualified Streamly.Internal.Data.Stream.StreamD.Type as D @@ -203,8 +204,8 @@ fromProducer sv = K.mkStream $ \st yld sng stp -> do -- function. -- {-# INLINE newFoldSVar #-} -newFoldSVar :: (K.IsStream t, MonadAsync m) - => State K.Stream m a -> (t m a -> m b) -> m (SVar K.Stream m a) +newFoldSVar :: MonadAsync m + => State K.Stream m a -> (SerialT m a -> m b) -> m (SVar K.Stream m a) newFoldSVar stt f = do -- Buffer size for the SVar is derived from the current state sv <- newParallelVar StopAny (adaptState stt) @@ -212,7 +213,7 @@ newFoldSVar stt f = do -- Add the producer thread-id to the SVar. liftIO myThreadId >>= modifyThread sv - void $ doFork (void $ f $ K.fromStream $ fromProducer sv) + void $ doFork (void $ f $ SerialT $ fromProducer sv) (svarMrun sv) (handleFoldException sv) return sv @@ -365,9 +366,9 @@ pushToFold sv a = do -- @ -- {-# INLINE teeToSVar #-} -teeToSVar :: (K.IsStream t, MonadAsync m) => - SVar K.Stream m a -> t m a -> t m a -teeToSVar svr m = K.mkStream $ \st yld sng stp -> do +teeToSVar :: MonadAsync m => + SVar K.Stream m a -> SerialT m a -> SerialT m a +teeToSVar svr (SerialT m) = SerialT $ K.mkStream $ \st yld sng stp -> do K.foldStreamShared st yld sng stp (go False m) where diff --git a/src/Streamly/Internal/Data/Stream/SVar/Generate.hs b/src/Streamly/Internal/Data/Stream/SVar/Generate.hs index 406eadcd25..f4a92e130e 100644 --- a/src/Streamly/Internal/Data/Stream/SVar/Generate.hs +++ b/src/Streamly/Internal/Data/Stream/SVar/Generate.hs @@ -19,8 +19,7 @@ module Streamly.Internal.Data.Stream.SVar.Generate ( -- * Write to SVar - toStreamVar - , toSVar + toSVar -- * Read from SVar -- $concurrentEval @@ -39,6 +38,7 @@ import Control.Monad.IO.Class (MonadIO(liftIO)) import Data.IORef (newIORef, readIORef, mkWeakIORef, writeIORef) import Data.Maybe (isNothing) import Streamly.Internal.Control.Concurrent (MonadAsync, captureMonadState) +import Streamly.Internal.Data.Stream.Serial (SerialT(..)) import Streamly.Internal.Data.Time.Clock (Clock(Monotonic), getTime) import System.Mem (performMajorGC) @@ -88,8 +88,8 @@ import Test.Inspection (inspect, hasNoTypeClassesExcept) -- XXX this errors out for Parallel/Ahead SVars -- | Write a stream to an 'SVar' in a non-blocking manner. The stream can then -- be read back from the SVar using 'fromSVar'. -toStreamVar :: MonadAsync m => SVar t m a -> t m a -> m () -toStreamVar sv m = do +toSVar :: MonadAsync m => SVar SerialT m a -> SerialT m a -> m () +toSVar sv m = do runIn <- captureMonadState liftIO $ enqueue sv (runIn, m) done <- allThreadsDone sv @@ -103,12 +103,6 @@ toStreamVar sv m = do Nothing -> pushWorker 0 sv Just _ -> pushWorker 1 sv --- | Write a stream to an 'SVar' in a non-blocking manner. The stream is --- evaluated concurrently as it is read back from the SVar using 'fromSVar'. --- -toSVar :: (K.IsStream t, MonadAsync m) => SVar K.Stream m a -> t m a -> m () -toSVar sv m = toStreamVar sv $ K.toStream m - ------------------------------------------------------------------------------- -- Read a stream from an SVar ------------------------------------------------------------------------------- @@ -191,16 +185,16 @@ inspect $ hasNoTypeClassesExcept 'fromStreamVar -- combinators. -- {-# INLINE fromSVar #-} -fromSVar :: (MonadAsync m, K.IsStream t) => SVar K.Stream m a -> t m a +fromSVar :: MonadAsync m => SVar K.Stream m a -> SerialT m a fromSVar sv = - K.mkStream $ \st yld sng stp -> do + SerialT $ K.mkStream $ \st yld sng stp -> do ref <- liftIO $ newIORef () _ <- liftIO $ mkWeakIORef ref hook -- We pass a copy of sv to fromStreamVar, so that we know that it has -- no other references, when that copy gets garbage collected "ref" -- will get garbage collected and our hook will be called. K.foldStreamShared st yld sng stp $ - K.fromStream $ fromStreamVar sv{svarRef = Just ref} + fromStreamVar sv{svarRef = Just ref} where hook = do diff --git a/src/Streamly/Internal/Data/Stream/Serial.hs b/src/Streamly/Internal/Data/Stream/Serial.hs index 3cec63fa0b..6f03cbf3b8 100644 --- a/src/Streamly/Internal/Data/Stream/Serial.hs +++ b/src/Streamly/Internal/Data/Stream/Serial.hs @@ -16,21 +16,28 @@ module Streamly.Internal.Data.Stream.Serial ( -- * Serial appending stream - SerialT + SerialT(..) , Serial - , K.serial - , fromSerial + , serial -- * Serial interleaving stream - , WSerialT + , WSerialT(..) , WSerial + , wSerialK , wSerial , wSerialFst , wSerialMin - , fromWSerial + , consMWSerial -- * Construction + , cons + , consM + , repeat , unfoldrM + , fromList + + -- * Elimination + , toList -- * Transformation , map @@ -39,8 +46,6 @@ module Streamly.Internal.Data.Stream.Serial -- * Deprecated , StreamT , InterleavedT - , (<=>) - , interleaving ) where @@ -67,18 +72,16 @@ import Text.Read ( Lexeme(Ident), lexP, parens, prec, readPrec, readListPrec , readListPrecDefault) import Streamly.Internal.BaseCompat ((#.), errorWithoutStackTrace, oneShot) -import Streamly.Internal.Data.Stream.StreamK.Type - (IsStream(..), adapt, Stream, mkStream, foldStream) import Streamly.Internal.Data.Maybe.Strict (Maybe'(..), toMaybe) +import Streamly.Internal.Data.Stream.StreamK.Type + (Stream, mkStream, foldStream) import qualified Streamly.Internal.Data.Stream.Prelude as P - (cmpBy, foldl', foldr, eqBy, fromList, toList) -import qualified Streamly.Internal.Data.Stream.StreamK as K (withLocal) -import qualified Streamly.Internal.Data.Stream.StreamK.Type as K -import qualified Streamly.Internal.Data.Stream.StreamD.Generate as D (unfoldrM) +import qualified Streamly.Internal.Data.Stream.StreamD.Generate as D import qualified Streamly.Internal.Data.Stream.StreamD.Type as D +import qualified Streamly.Internal.Data.Stream.StreamK.Type as K -import Prelude hiding (map, mapM, errorWithoutStackTrace) +import Prelude hiding (map, mapM, errorWithoutStackTrace, repeat) #include "Instances.hs" #include "inline.hs" @@ -136,24 +139,33 @@ type Serial = SerialT IO {-# DEPRECATED StreamT "Please use 'SerialT' instead." #-} type StreamT = SerialT --- | Fix the type of a polymorphic stream as 'SerialT'. --- --- /Since: 0.1.0 ("Streamly")/ +------------------------------------------------------------------------------ +-- Generation +------------------------------------------------------------------------------ + +{-# INLINE cons #-} +cons :: a -> SerialT m a -> SerialT m a +cons x (SerialT ms) = SerialT $ K.cons x ms + +{-# INLINE consM #-} +{-# SPECIALIZE consM :: IO a -> SerialT IO a -> SerialT IO a #-} +consM :: Monad m => m a -> SerialT m a -> SerialT m a +consM m (SerialT ms) = SerialT $ K.consM m ms + +-- | +-- Generate an infinite stream by repeating a pure value. -- --- @since 0.8.0 -fromSerial :: IsStream t => SerialT m a -> t m a -fromSerial = adapt +{-# INLINE_NORMAL repeat #-} +repeat :: Monad m => a -> SerialT m a +repeat = SerialT . D.toStreamK . D.repeat -{-# INLINE consMSerial #-} -{-# SPECIALIZE consMSerial :: IO a -> SerialT IO a -> SerialT IO a #-} -consMSerial :: Monad m => m a -> SerialT m a -> SerialT m a -consMSerial m ms = fromStream $ K.consMStream m (toStream ms) +------------------------------------------------------------------------------ +-- Combining +------------------------------------------------------------------------------ -instance IsStream SerialT where - toStream = getSerialT - fromStream = SerialT - consM = consMSerial - (|:) = consMSerial +{-# INLINE serial #-} +serial :: SerialT m a -> SerialT m a -> SerialT m a +serial = (<>) ------------------------------------------------------------------------------ -- Monad @@ -173,7 +185,7 @@ instance Monad m => Monad (SerialT m) where -- n binds, breakAfterSome, filterAllIn, state transformer (~2x) -- {-# INLINE (>>=) #-} - (>>=) = K.bindWith K.serial + (>>=) (SerialT m) f = SerialT $ K.bindWith K.serial m (getSerialT . f) {-# INLINE (>>) #-} (>>) = (*>) @@ -183,8 +195,8 @@ instance Monad m => Monad (SerialT m) where ------------------------------------------------------------------------------ {-# INLINE mapM #-} -mapM :: (IsStream t, Monad m) => (a -> m b) -> t m a -> t m b -mapM f m = D.fromStreamD $ D.mapM f $ D.toStreamD m +mapM :: Monad m => (a -> m b) -> SerialT m a -> SerialT m b +mapM f (SerialT m) = SerialT $ D.toStreamK $ D.mapM f $ D.fromStreamK m -- | -- @ @@ -200,23 +212,23 @@ mapM f m = D.fromStreamD $ D.mapM f $ D.toStreamD m -- -- @since 0.4.0 {-# INLINE map #-} -map :: (IsStream t, Monad m) => (a -> b) -> t m a -> t m b +map :: Monad m => (a -> b) -> SerialT m a -> SerialT m b map f = mapM (return . f) {-# INLINE apSerial #-} apSerial :: Monad m => SerialT m (a -> b) -> SerialT m a -> SerialT m b apSerial (SerialT m1) (SerialT m2) = - D.fromStreamD $ D.toStreamD m1 <*> D.toStreamD m2 + SerialT $ D.toStreamK $ D.fromStreamK m1 <*> D.fromStreamK m2 {-# INLINE apSequence #-} apSequence :: Monad m => SerialT m a -> SerialT m b -> SerialT m b apSequence (SerialT m1) (SerialT m2) = - D.fromStreamD $ D.toStreamD m1 *> D.toStreamD m2 + SerialT $ D.toStreamK $ D.fromStreamK m1 *> D.fromStreamK m2 {-# INLINE apDiscardSnd #-} apDiscardSnd :: Monad m => SerialT m a -> SerialT m b -> SerialT m a apDiscardSnd (SerialT m1) (SerialT m2) = - D.fromStreamD $ D.toStreamD m1 <* D.toStreamD m2 + SerialT $ D.toStreamK $ D.fromStreamK m1 <* D.fromStreamK m2 -- Note: we need to define all the typeclass operations because we want to -- INLINE them. @@ -285,7 +297,7 @@ TRAVERSABLE_INSTANCE(SerialT) -- @2@: -- -- >>> import Streamly.Prelude (wSerial) --- >>> Stream.toList $ Stream.fromList [(1,3),(1,4)] `wSerial` Stream.fromList [(2,3),(2,4)] +-- >>> Stream.toList $ Stream.fromList [(1,3),(1,4)] `Stream.wSerial` Stream.fromList [(2,3),(2,4)] -- [(1,3),(2,3),(1,4),(2,4)] -- -- The @W@ in the name stands for @wide@ or breadth wise scheduling in @@ -310,37 +322,10 @@ type WSerial = WSerialT IO {-# DEPRECATED InterleavedT "Please use 'WSerialT' instead." #-} type InterleavedT = WSerialT --- | Fix the type of a polymorphic stream as 'WSerialT'. --- --- /Since: 0.2.0 ("Streamly")/ --- --- @since 0.8.0 -fromWSerial :: IsStream t => WSerialT m a -> t m a -fromWSerial = adapt - --- | Same as 'fromWSerial'. --- --- @since 0.1.0 -{-# DEPRECATED interleaving "Please use fromWSerial instead." #-} -interleaving :: IsStream t => WSerialT m a -> t m a -interleaving = fromWSerial - +{-# INLINE consMWSerial #-} +{-# SPECIALIZE consMWSerial :: IO a -> WSerialT IO a -> WSerialT IO a #-} consMWSerial :: Monad m => m a -> WSerialT m a -> WSerialT m a -consMWSerial m ms = fromStream $ K.consMStream m (toStream ms) - -instance IsStream WSerialT where - toStream = getWSerialT - fromStream = WSerialT - - {-# INLINE consM #-} - {-# SPECIALIZE consM :: IO a -> WSerialT IO a -> WSerialT IO a #-} - consM :: Monad m => m a -> WSerialT m a -> WSerialT m a - consM = consMWSerial - - {-# INLINE (|:) #-} - {-# SPECIALIZE (|:) :: IO a -> WSerialT IO a -> WSerialT IO a #-} - (|:) :: Monad m => m a -> WSerialT m a -> WSerialT m a - (|:) = consMWSerial +consMWSerial m (WSerialT ms) = WSerialT $ K.consM m ms ------------------------------------------------------------------------------ -- Semigroup @@ -355,25 +340,17 @@ infixr 6 `wSerial` -- -- Similar combinators can be implemented using WAhead style. +{-# INLINE wSerialK #-} +wSerialK :: Stream m a -> Stream m a -> Stream m a +wSerialK m1 m2 = mkStream $ \st yld sng stp -> do + let stop = foldStream st yld sng stp m2 + single a = yld a m2 + yieldk a r = yld a (wSerialK m2 r) + foldStream st yieldk single stop m1 + -- | Interleaves two streams, yielding one element from each stream -- alternately. When one stream stops the rest of the other stream is used in -- the output stream. --- --- >>> import Streamly.Prelude (wSerial) --- >>> stream1 = Stream.fromList [1,2] --- >>> stream2 = Stream.fromList [3,4] --- >>> Stream.toList $ Stream.fromWSerial $ stream1 `wSerial` stream2 --- [1,3,2,4] --- --- Note, for singleton streams 'wSerial' and 'serial' are identical. --- --- Note that this operation cannot be used to fold a container of infinite --- streams but it can be used for very large streams as the state that it needs --- to maintain is proportional to the logarithm of the number of streams. --- --- @since 0.8.0 --- --- /Since: 0.2.0 ("Streamly")/ -- Scheduling Notes: -- @@ -387,19 +364,15 @@ infixr 6 `wSerial` -- each subexpression on the right. -- {-# INLINE wSerial #-} -wSerial :: IsStream t => t m a -> t m a -> t m a -wSerial m1 m2 = mkStream $ \st yld sng stp -> do - let stop = foldStream st yld sng stp m2 - single a = yld a m2 - yieldk a r = yld a (wSerial m2 r) - foldStream st yieldk single stop m1 +wSerial :: WSerialT m a -> WSerialT m a -> WSerialT m a +wSerial (WSerialT m1) (WSerialT m2) = WSerialT $ wSerialK m1 m2 -- | Like `wSerial` but stops interleaving as soon as the first stream stops. -- -- @since 0.7.0 -{-# INLINE wSerialFst #-} -wSerialFst :: IsStream t => t m a -> t m a -> t m a -wSerialFst m1 m2 = mkStream $ \st yld sng stp -> do +{-# INLINE wSerialFstK #-} +wSerialFstK :: Stream m a -> Stream m a -> Stream m a +wSerialFstK m1 m2 = mkStream $ \st yld sng stp -> do let yieldFirst a r = yld a (yieldSecond r m2) in foldStream st yieldFirst sng stp m1 @@ -408,49 +381,47 @@ wSerialFst m1 m2 = mkStream $ \st yld sng stp -> do yieldSecond s1 s2 = mkStream $ \st yld sng stp -> do let stop = foldStream st yld sng stp s1 single a = yld a s1 - yieldk a r = yld a (wSerial s1 r) + yieldk a r = yld a (wSerialK s1 r) in foldStream st yieldk single stop s2 +{-# INLINE wSerialFst #-} +wSerialFst :: WSerialT m a -> WSerialT m a -> WSerialT m a +wSerialFst (WSerialT m1) (WSerialT m2) = WSerialT $ wSerialFstK m1 m2 + -- | Like `wSerial` but stops interleaving as soon as any of the two streams -- stops. -- -- @since 0.7.0 -{-# INLINE wSerialMin #-} -wSerialMin :: IsStream t => t m a -> t m a -> t m a -wSerialMin m1 m2 = mkStream $ \st yld _ stp -> do +{-# INLINE wSerialMinK #-} +wSerialMinK :: Stream m a -> Stream m a -> Stream m a +wSerialMinK m1 m2 = mkStream $ \st yld _ stp -> do let stop = stp -- "single a" is defined as "yld a (wSerialMin m2 K.nil)" instead of -- "sng a" to keep the behaviour consistent with the yield continuation. - single a = yld a (wSerialMin m2 K.nil) - yieldk a r = yld a (wSerialMin m2 r) + single a = yld a (wSerialMinK m2 K.nil) + yieldk a r = yld a (wSerialMinK m2 r) foldStream st yieldk single stop m1 +{-# INLINE wSerialMin #-} +wSerialMin :: WSerialT m a -> WSerialT m a -> WSerialT m a +wSerialMin (WSerialT m1) (WSerialT m2) = WSerialT $ wSerialMinK m1 m2 + instance Semigroup (WSerialT m a) where (<>) = wSerial -infixr 5 <=> - --- | Same as 'wSerial'. --- --- @since 0.1.0 -{-# DEPRECATED (<=>) "Please use 'wSerial' instead." #-} -{-# INLINE (<=>) #-} -(<=>) :: IsStream t => t m a -> t m a -> t m a -(<=>) = wSerial - ------------------------------------------------------------------------------ -- Monoid ------------------------------------------------------------------------------ instance Monoid (WSerialT m a) where - mempty = K.nil + mempty = WSerialT K.nil mappend = (<>) {-# INLINE apWSerial #-} apWSerial :: Monad m => WSerialT m (a -> b) -> WSerialT m a -> WSerialT m b apWSerial (WSerialT m1) (WSerialT m2) = - let f x1 = K.concatMapBy wSerial (pure . x1) m2 - in WSerialT $ K.concatMapBy wSerial f m1 + let f x1 = K.concatMapWith wSerialK (pure . x1) m2 + in WSerialT $ K.concatMapWith wSerialK f m1 instance Monad m => Applicative (WSerialT m) where {-# INLINE pure #-} @@ -465,7 +436,7 @@ instance Monad m => Applicative (WSerialT m) where instance Monad m => Monad (WSerialT m) where return = pure {-# INLINE (>>=) #-} - (>>=) = K.bindWith wSerial + (>>=) (WSerialT m) f = WSerialT $ K.bindWith wSerialK m (getWSerialT . f) ------------------------------------------------------------------------------ -- Other instances @@ -503,5 +474,5 @@ TRAVERSABLE_INSTANCE(WSerialT) -- /Pre-release/ -- {-# INLINE unfoldrM #-} -unfoldrM :: (IsStream t, Monad m) => (b -> m (Maybe (a, b))) -> b -> t m a -unfoldrM step seed = D.fromStreamD (D.unfoldrM step seed) +unfoldrM :: Monad m => (b -> m (Maybe (a, b))) -> b -> SerialT m a +unfoldrM step seed = SerialT $ D.toStreamK (D.unfoldrM step seed) diff --git a/src/Streamly/Internal/Data/Stream/StreamD/Eliminate.hs b/src/Streamly/Internal/Data/Stream/StreamD/Eliminate.hs index 4dcb635395..818c53bdfc 100644 --- a/src/Streamly/Internal/Data/Stream/StreamD/Eliminate.hs +++ b/src/Streamly/Internal/Data/Stream/StreamD/Eliminate.hs @@ -228,19 +228,6 @@ parse_ (PRD.Parser pstep initial extract) stream@(Stream step state) = do -- Specialized Folds ------------------------------------------------------------------------------ --- | Run a streaming composition, discard the results. -{-# INLINE_LATE drain #-} -drain :: Monad m => Stream m a -> m () --- drain = foldrM (\_ xs -> xs) (return ()) -drain (Stream step state) = go SPEC state - where - go !_ st = do - r <- step defState st - case r of - Yield _ s -> go SPEC s - Skip s -> go SPEC s - Stop -> return () - {-# INLINE_NORMAL null #-} null :: Monad m => Stream m a -> m Bool null = foldrM (\_ _ -> return False) (return True) diff --git a/src/Streamly/Internal/Data/Stream/StreamD/Generate.hs b/src/Streamly/Internal/Data/Stream/StreamD/Generate.hs index ea9b9a8e61..deef3efba8 100644 --- a/src/Streamly/Internal/Data/Stream/StreamD/Generate.hs +++ b/src/Streamly/Internal/Data/Stream/StreamD/Generate.hs @@ -82,8 +82,6 @@ module Streamly.Internal.Data.Stream.StreamD.Generate -- * Conversions , fromStreamK , toStreamK - , fromStreamD - , toStreamD ) where diff --git a/src/Streamly/Internal/Data/Stream/StreamD/Transform.hs b/src/Streamly/Internal/Data/Stream/StreamD/Transform.hs index e2613f0412..ed1c994d11 100644 --- a/src/Streamly/Internal/Data/Stream/StreamD/Transform.hs +++ b/src/Streamly/Internal/Data/Stream/StreamD/Transform.hs @@ -99,7 +99,7 @@ module Streamly.Internal.Data.Stream.StreamD.Transform -- * Reordering -- | Produce strictly the same set but reordered. , reverse - , reverse' + -- , reverse' -- * Position Indexing , indexed @@ -129,7 +129,6 @@ import Control.Monad.IO.Class (MonadIO(..)) import Control.Monad.Trans.Class (MonadTrans(lift)) import Data.IORef (newIORef, mkWeakIORef) import Data.Maybe (fromJust, isJust) -import Foreign.Storable (Storable(..)) import GHC.Types (SPEC(..)) import qualified Control.Monad.Catch as MC @@ -140,13 +139,10 @@ import Streamly.Internal.Data.SVar.Type (defState, adaptState) import Streamly.Internal.Data.Time.Clock (Clock(Monotonic), getTime) import Streamly.Internal.Data.Time.Units (TimeUnit64, toRelTime64, diffAbsTime64) -import Streamly.Internal.System.IO (defaultChunkSize) -import qualified Streamly.Internal.Data.Array.Foreign.Type as A import qualified Streamly.Internal.Data.Fold.Type as FL import qualified Streamly.Internal.Data.IORef.Prim as Prim import qualified Streamly.Internal.Data.Pipe.Type as Pipe -import qualified Streamly.Internal.Data.Stream.StreamK as K import Prelude hiding ( drop, dropWhile, filter, map, mapM, reverse @@ -1100,12 +1096,12 @@ reverse m = Stream step Nothing step _ (Just []) = return Stop -- Much faster reverse for Storables +{- {-# INLINE_NORMAL reverse' #-} reverse' :: forall m a. (MonadIO m, Storable a) => Stream m a -> Stream m a -{- -- This commented implementation copies the whole stream into one single array -- and then streams from that array, this has exactly the same performance as --- the chunked code that follows. Though this could be problematic due to +-- the chunked code in IsStream.Common.reverse' . Though this could be problematic due to -- unbounded large allocations. However, if we use an idiomatic implementation -- of arraysOf instead of the custom implementation then the chunked code -- becomes worse by 6 times. Need to investigate if that can be improved. @@ -1130,12 +1126,6 @@ reverse' m = Stream step Nothing next = p `plusPtr` negate (sizeOf (undefined :: a)) return $ Yield x (Just (start, next)) -} -reverse' = - A.flattenArraysRev -- unfoldMany A.readRev - . fromStreamK - . K.reverse - . toStreamK - . A.arraysOf defaultChunkSize ------------------------------------------------------------------------------ -- Position Indexing diff --git a/src/Streamly/Internal/Data/Stream/StreamD/Type.hs b/src/Streamly/Internal/Data/Stream/StreamD/Type.hs index a3e9a47406..aeea184187 100644 --- a/src/Streamly/Internal/Data/Stream/StreamD/Type.hs +++ b/src/Streamly/Internal/Data/Stream/StreamD/Type.hs @@ -39,8 +39,6 @@ module Streamly.Internal.Data.Stream.StreamD.Type -- * Conversions From/To , fromStreamK , toStreamK - , toStreamD - , fromStreamD -- * Running a 'Fold' , fold @@ -59,6 +57,9 @@ module Streamly.Internal.Data.Stream.StreamD.Type , foldlx' , foldlMx' + -- * Special Folds + , drain + -- * To Containers , toList @@ -239,7 +240,7 @@ fromStreamK = Stream step toStreamK :: Monad m => Stream m a -> K.Stream m a toStreamK (Stream step state) = go state where - go st = K.mkStream $ \gst yld _ stp -> + go st = K.MkStream $ \gst yld _ stp -> let go' ss = do r <- step gst ss case r of @@ -255,16 +256,6 @@ toStreamK (Stream step state) = go state forall s. fromStreamK (toStreamK s) = s #-} #endif --- XXX Rename to toStream or move to some IsStream common module -{-# INLINE fromStreamD #-} -fromStreamD :: (K.IsStream t, Monad m) => Stream m a -> t m a -fromStreamD = K.fromStream . toStreamK - --- XXX Rename to toStream or move to some IsStream common module -{-# INLINE toStreamD #-} -toStreamD :: (K.IsStream t, Monad m) => t m a -> Stream m a -toStreamD = fromStreamK . K.toStream - ------------------------------------------------------------------------------ -- Running a 'Fold' ------------------------------------------------------------------------------ @@ -443,6 +434,23 @@ foldlM' fstep mbegin (Stream step state) = do foldl' :: Monad m => (b -> a -> b) -> b -> Stream m a -> m b foldl' fstep begin = foldlM' (\b a -> return (fstep b a)) (return begin) +------------------------------------------------------------------------------ +-- Special folds +------------------------------------------------------------------------------ + +-- | Run a streaming composition, discard the results. +{-# INLINE_LATE drain #-} +drain :: Monad m => Stream m a -> m () +-- drain = foldrM (\_ xs -> xs) (return ()) +drain (Stream step state) = go SPEC state + where + go !_ st = do + r <- step defState st + case r of + Yield _ s -> go SPEC s + Skip s -> go SPEC s + Stop -> return () + ------------------------------------------------------------------------------ -- To Containers ------------------------------------------------------------------------------ diff --git a/src/Streamly/Internal/Data/Stream/StreamK.hs b/src/Streamly/Internal/Data/Stream/StreamK.hs index f44e5f6e4b..7765cfd242 100644 --- a/src/Streamly/Internal/Data/Stream/StreamK.hs +++ b/src/Streamly/Internal/Data/Stream/StreamK.hs @@ -21,12 +21,8 @@ -- module Streamly.Internal.Data.Stream.StreamK ( - -- * A class for streams - IsStream (..) - , adapt - -- * The stream type - , Stream(..) + Stream(..) -- * Construction Primitives , mkStream @@ -65,15 +61,12 @@ module Streamly.Internal.Data.Stream.StreamK , fromEffect , fromFoldable , fromList - , fromStreamK -- * foldr/build , foldrS , foldrSM , buildS - , buildM , augmentS - , augmentSM -- * Elimination -- ** General Folds @@ -116,7 +109,6 @@ module Streamly.Internal.Data.Stream.StreamK -- ** Conversions , toList - , toStreamK , hoist -- * Transformation @@ -134,7 +126,6 @@ module Streamly.Internal.Data.Stream.StreamK -- ** Mapping , map , mapM - , mapMSerial , sequence -- ** Inserting @@ -160,7 +151,7 @@ module Streamly.Internal.Data.Stream.StreamK , mergeByM -- ** Nesting - , concatMapBy + , concatMapWith , concatMap , bindWith , concatPairsWith @@ -176,24 +167,16 @@ module Streamly.Internal.Data.Stream.StreamK , serial -- * Utilities - , consMStream + , consM , withLocal , mfix - - -- * Deprecated - , Streaming -- deprecated - , once -- deprecated ) where import Control.Monad.Trans.Class (MonadTrans(lift)) import Control.Monad (void, join) -import Control.Monad.Reader.Class (MonadReader(..)) -import Data.Function (fix) -import Streamly.Internal.Control.Concurrent (MonadAsync) import Streamly.Internal.Data.SVar.Type (adaptState, defState) -import qualified Prelude import qualified Streamly.Internal.Data.Fold.Type as FL import Prelude @@ -207,41 +190,13 @@ import Streamly.Internal.Data.Stream.StreamK.Type -- $setup -- >>> :m -------------------------------------------------------------------------------- --- Deconstruction -------------------------------------------------------------------------------- - -{-# INLINE uncons #-} -uncons :: (IsStream t, Monad m) => t m a -> m (Maybe (a, t m a)) -uncons m = - let stop = return Nothing - single a = return (Just (a, nil)) - yieldk a r = return (Just (a, r)) - in foldStream defState yieldk single stop m - ------------------------------------------------------------------------------- -- Generation ------------------------------------------------------------------------------- -{-# INLINE unfoldr #-} -unfoldr :: IsStream t => (b -> Maybe (a, b)) -> b -> t m a -unfoldr next s0 = build $ \yld stp -> - let go s = - case next s of - Just (a, b) -> yld a (go b) - Nothing -> stp - in go s0 - {-# INLINE unfoldrM #-} -unfoldrM :: (IsStream t, MonadAsync m) => (b -> m (Maybe (a, b))) -> b -> t m a -unfoldrM step = go - where - go s = sharedM $ \yld _ stp -> do - r <- step s - case r of - Just (a, b) -> yld a (go b) - Nothing -> stp - +unfoldrM :: Monad m => (b -> m (Maybe (a, b))) -> b -> Stream m a +unfoldrM = unfoldrMWith consM {- -- Generalization of concurrent streams/SVar via unfoldr. -- @@ -255,7 +210,8 @@ unfoldrM step = go -- concurrent version could be async or ahead etc. Depending on how we queue -- back the feedback portion b, it could be DFS or BFS style. -- -unfoldrA :: (IsStream t, MonadAsync m) => (b -> Maybe (m a, b)) -> b -> t m a +unfoldrA :: (IsStream t, MonadAsync m) => + (b -> Maybe (m a, b)) -> b -> Stream m a unfoldrA = undefined -} @@ -263,110 +219,58 @@ unfoldrA = undefined -- Special generation ------------------------------------------------------------------------------- --- | Same as fromEffect --- --- @since 0.2.0 -{-# DEPRECATED once "Please use fromEffect instead." #-} -{-# INLINE once #-} -once :: (Monad m, IsStream t) => m a -> t m a -once = fromEffect - --- | --- @ --- repeatM = fix . cons --- repeatM = cycle1 . fromPure --- @ --- --- Generate an infinite stream by repeating a monadic value. --- --- /Pre-release/ -repeatM :: (IsStream t, MonadAsync m) => m a -> t m a -repeatM = go - where go m = m |: go m - --- Generate an infinite stream by repeating a pure value. --- --- /Pre-release/ -{-# INLINE repeat #-} -repeat :: IsStream t => a -> t m a -repeat a = let x = cons a x in x +repeatM :: Monad m => m a -> Stream m a +repeatM = repeatMWith consM {-# INLINE replicateM #-} -replicateM :: (IsStream t, MonadAsync m) => Int -> m a -> t m a -replicateM n m = go n - where - go cnt = if cnt <= 0 then nil else m |: go (cnt - 1) - +replicateM :: Monad m => Int -> m a -> Stream m a +replicateM = replicateMWith consM {-# INLINE replicate #-} -replicate :: IsStream t => Int -> a -> t m a +replicate :: Int -> a -> Stream m a replicate n a = go n where go cnt = if cnt <= 0 then nil else a `cons` go (cnt - 1) {-# INLINE fromIndicesM #-} -fromIndicesM :: (IsStream t, MonadAsync m) => (Int -> m a) -> t m a -fromIndicesM gen = go 0 - where - go i = mkStream $ \st stp sng yld -> do - foldStreamShared st stp sng yld (gen i |: go (i + 1)) - +fromIndicesM :: Monad m => (Int -> m a) -> Stream m a +fromIndicesM = fromIndicesMWith consM {-# INLINE fromIndices #-} -fromIndices :: IsStream t => (Int -> a) -> t m a +fromIndices :: (Int -> a) -> Stream m a fromIndices gen = go 0 where go n = gen n `cons` go (n + 1) {-# INLINE iterate #-} -iterate :: IsStream t => (a -> a) -> a -> t m a +iterate :: (a -> a) -> a -> Stream m a iterate step = go where go !s = cons s (go (step s)) {-# INLINE iterateM #-} -iterateM :: (IsStream t, MonadAsync m) => (a -> m a) -> m a -> t m a -iterateM step = go - where - go s = mkStream $ \st stp sng yld -> do - !next <- s - foldStreamShared st stp sng yld (return next |: go (step next)) +iterateM :: Monad m => (a -> m a) -> m a -> Stream m a +iterateM = iterateMWith consM ------------------------------------------------------------------------------- -- Conversions ------------------------------------------------------------------------------- --- | --- @ --- fromFoldable = 'Prelude.foldr' 'cons' 'nil' --- @ --- --- Construct a stream from a 'Foldable' containing pure values: --- --- @since 0.2.0 -{-# INLINE fromFoldable #-} -fromFoldable :: (IsStream t, Foldable f) => f a -> t m a -fromFoldable = Prelude.foldr cons nil - {-# INLINE fromList #-} -fromList :: IsStream t => [a] -> t m a +fromList :: [a] -> Stream m a fromList = fromFoldable -{-# INLINE fromStreamK #-} -fromStreamK :: IsStream t => Stream m a -> t m a -fromStreamK = fromStream - ------------------------------------------------------------------------------- -- Elimination by Folding ------------------------------------------------------------------------------- -- | Lazy right associative fold. {-# INLINE foldr #-} -foldr :: (IsStream t, Monad m) => (a -> b -> b) -> b -> t m a -> m b +foldr :: Monad m => (a -> b -> b) -> b -> Stream m a -> m b foldr step acc = foldrM (\x xs -> xs >>= \b -> return (step x b)) (return acc) -- | Right associative fold to an arbitrary transformer monad. {-# INLINE foldrT #-} -foldrT :: (IsStream t, Monad m, Monad (s m), MonadTrans s) - => (a -> s m b -> s m b) -> s m b -> t m a -> s m b +foldrT :: (Monad m, Monad (s m), MonadTrans s) + => (a -> s m b -> s m b) -> s m b -> Stream m a -> s m b foldrT step final = go where go m1 = do @@ -376,7 +280,7 @@ foldrT step final = go Nothing -> final {-# INLINE foldr1 #-} -foldr1 :: (IsStream t, Monad m) => (a -> a -> a) -> t m a -> m (Maybe a) +foldr1 :: Monad m => (a -> a -> a) -> Stream m a -> m (Maybe a) foldr1 step m = do r <- uncons m case r of @@ -392,8 +296,8 @@ foldr1 step m = do -- XXX replace the recursive "go" with explicit continuations. -- | Like 'foldx', but with a monadic step function. {-# INLINABLE foldlMx' #-} -foldlMx' :: (IsStream t, Monad m) - => (x -> a -> m x) -> m x -> (x -> m b) -> t m a -> m b +foldlMx' :: Monad m + => (x -> a -> m x) -> m x -> (x -> m b) -> Stream m a -> m b foldlMx' step begin done = go begin where go !acc m1 = @@ -403,7 +307,7 @@ foldlMx' step begin done = go begin in foldStream defState yieldk single stop m1 {-# INLINABLE fold #-} -fold :: (IsStream t, Monad m) => FL.Fold m a b -> t m a -> m b +fold :: Monad m => FL.Fold m a b -> Stream m a -> m b fold (FL.Fold step begin done) m = do res <- begin case res of @@ -425,25 +329,13 @@ fold (FL.Fold step begin done) m = do -- | Like 'foldl'' but with a monadic step function. {-# INLINE foldlM' #-} -foldlM' :: (IsStream t, Monad m) => (b -> a -> m b) -> m b -> t m a -> m b +foldlM' :: Monad m => (b -> a -> m b) -> m b -> Stream m a -> m b foldlM' step begin = foldlMx' step begin return --- | Lazy left fold to a stream. -{-# INLINE foldlS #-} -foldlS :: IsStream t => (t m b -> a -> t m b) -> t m b -> t m a -> t m b -foldlS step = go - where - go acc rest = mkStream $ \st yld sng stp -> - let run x = foldStream st yld sng stp x - stop = run acc - single a = run $ step acc a - yieldk a r = run $ go (step acc a) r - in foldStream (adaptState st) yieldk single stop rest - -- | Lazy left fold to an arbitrary transformer monad. {-# INLINE foldlT #-} -foldlT :: (IsStream t, Monad m, Monad (s m), MonadTrans s) - => (s m b -> a -> s m b) -> s m b -> t m a -> s m b +foldlT :: (Monad m, Monad (s m), MonadTrans s) + => (s m b -> a -> s m b) -> s m b -> Stream m a -> s m b foldlT step = go where go acc m1 = do @@ -456,39 +348,8 @@ foldlT step = go -- Specialized folds ------------------------------------------------------------------------------ --- XXX use foldrM to implement folds where possible --- XXX This (commented) definition of drain and mapM_ perform much better on --- some benchmarks but worse on others. Need to investigate why, may there is --- an optimization opportunity that we can exploit. --- drain = foldrM (\_ xs -> return () >> xs) (return ()) - --- | --- > drain = foldl' (\_ _ -> ()) () --- > drain = mapM_ (\_ -> return ()) -{-# INLINE drain #-} -drain :: (Monad m, IsStream t) => t m a -> m () -drain = foldrM (\_ xs -> xs) (return ()) -{- -drain = go - where - go m1 = - let stop = return () - single _ = return () - yieldk _ r = go r - in foldStream defState yieldk single stop m1 --} - -{-# INLINE null #-} -null :: (IsStream t, Monad m) => t m a -> m Bool --- null = foldrM (\_ _ -> return True) (return False) -null m = - let stop = return True - single _ = return False - yieldk _ _ = return False - in foldStream defState yieldk single stop m - {-# INLINE head #-} -head :: (IsStream t, Monad m) => t m a -> m (Maybe a) +head :: Monad m => Stream m a -> m (Maybe a) -- head = foldrM (\x _ -> return $ Just x) (return Nothing) head m = let stop = return Nothing @@ -496,112 +357,8 @@ head m = yieldk a _ = return (Just a) in foldStream defState yieldk single stop m -{-# INLINE tail #-} -tail :: (IsStream t, Monad m) => t m a -> m (Maybe (t m a)) -tail m = - let stop = return Nothing - single _ = return $ Just nil - yieldk _ r = return $ Just r - in foldStream defState yieldk single stop m - -{-# INLINE headPartial #-} -headPartial :: (IsStream t, Monad m) => t m a -> m a -headPartial = foldrM (\x _ -> return x) (error "head of nil") - -{-# INLINE tailPartial #-} -tailPartial :: IsStream t => t m a -> t m a -tailPartial m = mkStream $ \st yld sng stp -> - let stop = error "tail of nil" - single _ = stp - yieldk _ r = foldStream st yld sng stp r - in foldStream st yieldk single stop m - --- | We can define cyclic structures using @let@: --- --- >>> let (a, b) = ([1, b], head a) in (a, b) --- ([1,1],1) --- --- The function @fix@ defined as: --- --- > fix f = let x = f x in x --- --- ensures that the argument of a function and its output refer to the same --- lazy value @x@ i.e. the same location in memory. Thus @x@ can be defined --- in terms of itself, creating structures with cyclic references. --- --- >>> import Data.Function (fix) --- >>> f ~(a, b) = ([1, b], head a) --- >>> fix f --- ([1,1],1) --- --- 'Control.Monad.mfix' is essentially the same as @fix@ but for monadic --- values. --- --- Using 'mfix' for streams we can construct a stream in which each element of --- the stream is defined in a cyclic fashion. The argument of the function --- being fixed represents the current element of the stream which is being --- returned by the stream monad. Thus, we can use the argument to construct --- itself. --- --- In the following example, the argument @action@ of the function @f@ --- represents the tuple @(x,y)@ returned by it in a given iteration. We define --- the first element of the tuple in terms of the second. --- --- @ --- import Streamly.Internal.Data.Stream.IsStream as Stream --- import System.IO.Unsafe (unsafeInterleaveIO) --- --- main = do --- Stream.mapM_ print $ Stream.mfix f --- --- where --- --- f action = do --- let incr n act = fmap ((+n) . snd) $ unsafeInterleaveIO act --- x <- Stream.fromListM [incr 1 action, incr 2 action] --- y <- Stream.fromList [4,5] --- return (x, y) --- @ --- --- Note: you cannot achieve this by just changing the order of the monad --- statements because that would change the order in which the stream elements --- are generated. --- --- Note that the function @f@ must be lazy in its argument, that's why we use --- 'unsafeInterleaveIO' on @action@ because IO monad is strict. --- --- /Pre-release/ - -mfix :: (IsStream t, Monad m) => (m a -> t m a) -> t m a -mfix f = mkStream $ \st yld sng stp -> - let single a = foldStream st yld sng stp $ a `cons` ys - yieldk a _ = foldStream st yld sng stp $ a `cons` ys - in foldStream st yieldk single stp xs - - where - - -- fix the head element of the stream - xs = fix (f . headPartial) - - -- now fix the tail recursively - ys = mfix (tailPartial . f) - -{-# INLINE init #-} -init :: (IsStream t, Monad m) => t m a -> m (Maybe (t m a)) -init = go1 - where - go1 m1 = do - r <- uncons m1 - case r of - Nothing -> return Nothing - Just (h, t) -> return . Just $ go h t - go p m1 = mkStream $ \_ yld sng stp -> - let single _ = sng p - yieldk a x = yld p $ go a x - in foldStream defState yieldk single stp m1 - {-# INLINE elem #-} -elem :: (IsStream t, Monad m, Eq a) => a -> t m a -> m Bool +elem :: (Monad m, Eq a) => a -> Stream m a -> m Bool elem e = go where go m1 = @@ -611,7 +368,7 @@ elem e = go in foldStream defState yieldk single stop m1 {-# INLINE notElem #-} -notElem :: (IsStream t, Monad m, Eq a) => a -> t m a -> m Bool +notElem :: (Monad m, Eq a) => a -> Stream m a -> m Bool notElem e = go where go m1 = @@ -621,7 +378,7 @@ notElem e = go in foldStream defState yieldk single stop m1 {-# INLINABLE all #-} -all :: (IsStream t, Monad m) => (a -> Bool) -> t m a -> m Bool +all :: Monad m => (a -> Bool) -> Stream m a -> m Bool all p = go where go m1 = @@ -632,7 +389,7 @@ all p = go in foldStream defState yieldk single (return True) m1 {-# INLINABLE any #-} -any :: (IsStream t, Monad m) => (a -> Bool) -> t m a -> m Bool +any :: Monad m => (a -> Bool) -> Stream m a -> m Bool any p = go where go m1 = @@ -644,11 +401,11 @@ any p = go -- | Extract the last element of the stream, if any. {-# INLINE last #-} -last :: (IsStream t, Monad m) => t m a -> m (Maybe a) +last :: Monad m => Stream m a -> m (Maybe a) last = foldlx' (\_ y -> Just y) Nothing id {-# INLINE minimum #-} -minimum :: (IsStream t, Monad m, Ord a) => t m a -> m (Maybe a) +minimum :: (Monad m, Ord a) => Stream m a -> m (Maybe a) minimum = go Nothing where go Nothing m1 = @@ -671,8 +428,8 @@ minimum = go Nothing {-# INLINE minimumBy #-} minimumBy - :: (IsStream t, Monad m) - => (a -> a -> Ordering) -> t m a -> m (Maybe a) + :: (Monad m) + => (a -> a -> Ordering) -> Stream m a -> m (Maybe a) minimumBy cmp = go Nothing where go Nothing m1 = @@ -692,7 +449,7 @@ minimumBy cmp = go Nothing in foldStream defState yieldk single stop m1 {-# INLINE maximum #-} -maximum :: (IsStream t, Monad m, Ord a) => t m a -> m (Maybe a) +maximum :: (Monad m, Ord a) => Stream m a -> m (Maybe a) maximum = go Nothing where go Nothing m1 = @@ -714,7 +471,7 @@ maximum = go Nothing in foldStream defState yieldk single stop m1 {-# INLINE maximumBy #-} -maximumBy :: (IsStream t, Monad m) => (a -> a -> Ordering) -> t m a -> m (Maybe a) +maximumBy :: Monad m => (a -> a -> Ordering) -> Stream m a -> m (Maybe a) maximumBy cmp = go Nothing where go Nothing m1 = @@ -734,7 +491,7 @@ maximumBy cmp = go Nothing in foldStream defState yieldk single stop m1 {-# INLINE (!!) #-} -(!!) :: (IsStream t, Monad m) => t m a -> Int -> m (Maybe a) +(!!) :: Monad m => Stream m a -> Int -> m (Maybe a) m !! i = go i m where go n m1 = @@ -746,7 +503,7 @@ m !! i = go i m in foldStream defState yieldk single (return Nothing) m1 {-# INLINE lookup #-} -lookup :: (IsStream t, Monad m, Eq a) => a -> t m (a, b) -> m (Maybe b) +lookup :: (Monad m, Eq a) => a -> Stream m (a, b) -> m (Maybe b) lookup e = go where go m1 = @@ -757,7 +514,7 @@ lookup e = go in foldStream defState yieldk single (return Nothing) m1 {-# INLINE findM #-} -findM :: (IsStream t, Monad m) => (a -> m Bool) -> t m a -> m (Maybe a) +findM :: Monad m => (a -> m Bool) -> Stream m a -> m (Maybe a) findM p = go where go m1 = @@ -770,11 +527,11 @@ findM p = go in foldStream defState yieldk single (return Nothing) m1 {-# INLINE find #-} -find :: (IsStream t, Monad m) => (a -> Bool) -> t m a -> m (Maybe a) +find :: Monad m => (a -> Bool) -> Stream m a -> m (Maybe a) find p = findM (return . p) {-# INLINE findIndices #-} -findIndices :: IsStream t => (a -> Bool) -> t m a -> t m Int +findIndices :: (a -> Bool) -> Stream m a -> Stream m Int findIndices p = go 0 where go offset m1 = mkStream $ \st yld sng stp -> @@ -792,7 +549,7 @@ findIndices p = go 0 -- | Apply a monadic action to each element of the stream and discard the -- output of the action. {-# INLINE mapM_ #-} -mapM_ :: (IsStream t, Monad m) => (a -> m b) -> t m a -> m () +mapM_ :: Monad m => (a -> m b) -> Stream m a -> m () mapM_ f = go where go m1 = @@ -801,22 +558,22 @@ mapM_ f = go yieldk a r = f a >> go r in foldStream defState yieldk single stop m1 +{-# INLINE mapM #-} +mapM :: Monad m => (a -> m b) -> Stream m a -> Stream m b +mapM = mapMWith consM + ------------------------------------------------------------------------------ -- Converting folds ------------------------------------------------------------------------------ {-# INLINABLE toList #-} -toList :: (IsStream t, Monad m) => t m a -> m [a] +toList :: Monad m => Stream m a -> m [a] toList = foldr (:) [] -{-# INLINE toStreamK #-} -toStreamK :: Stream m a -> Stream m a -toStreamK = id - -- Based on suggestions by David Feuer and Pranay Sashank {-# INLINE hoist #-} -hoist :: (IsStream t, Monad m, Monad n) - => (forall x. m x -> n x) -> t m a -> t n a +hoist :: (Monad m, Monad n) + => (forall x. m x -> n x) -> Stream m a -> Stream n a hoist f str = mkStream $ \st yld sng stp -> let single = return . sng @@ -830,7 +587,7 @@ hoist f str = ------------------------------------------------------------------------------- {-# INLINE scanlx' #-} -scanlx' :: IsStream t => (x -> a -> x) -> x -> (x -> b) -> t m a -> t m b +scanlx' :: (x -> a -> x) -> x -> (x -> b) -> Stream m a -> Stream m b scanlx' step begin done m = cons (done begin) $ go m begin where @@ -842,7 +599,7 @@ scanlx' step begin done m = in foldStream (adaptState st) yieldk single stp m1 {-# INLINE scanl' #-} -scanl' :: IsStream t => (b -> a -> b) -> b -> t m a -> t m b +scanl' :: (b -> a -> b) -> b -> Stream m a -> Stream m b scanl' step begin = scanlx' step begin id ------------------------------------------------------------------------------- @@ -850,7 +607,7 @@ scanl' step begin = scanlx' step begin id ------------------------------------------------------------------------------- {-# INLINE filter #-} -filter :: IsStream t => (a -> Bool) -> t m a -> t m a +filter :: (a -> Bool) -> Stream m a -> Stream m a filter p = go where go m1 = mkStream $ \st yld sng stp -> @@ -861,7 +618,7 @@ filter p = go in foldStream st yieldk single stp m1 {-# INLINE take #-} -take :: IsStream t => Int -> t m a -> t m a +take :: Int -> Stream m a -> Stream m a take = go where go n1 m1 = mkStream $ \st yld sng stp -> @@ -871,7 +628,7 @@ take = go else foldStream st yieldk sng stp m1 {-# INLINE takeWhile #-} -takeWhile :: IsStream t => (a -> Bool) -> t m a -> t m a +takeWhile :: (a -> Bool) -> Stream m a -> Stream m a takeWhile p = go where go m1 = mkStream $ \st yld sng stp -> @@ -882,8 +639,8 @@ takeWhile p = go in foldStream st yieldk single stp m1 {-# INLINE drop #-} -drop :: IsStream t => Int -> t m a -> t m a -drop n m = fromStream $ unShare (go n (toStream m)) +drop :: Int -> Stream m a -> Stream m a +drop n m = unShare (go n m) where go n1 m1 = mkStream $ \st yld sng stp -> let single _ = stp @@ -894,7 +651,7 @@ drop n m = fromStream $ unShare (go n (toStream m)) else foldStreamShared st yieldk single stp m1 {-# INLINE dropWhile #-} -dropWhile :: IsStream t => (a -> Bool) -> t m a -> t m a +dropWhile :: (a -> Bool) -> Stream m a -> Stream m a dropWhile p = go where go m1 = mkStream $ \st yld sng stp -> @@ -911,12 +668,12 @@ dropWhile p = go -- Be careful when modifying this, this uses a consM (|:) deliberately to allow -- other stream types to overload it. {-# INLINE sequence #-} -sequence :: (IsStream t, MonadAsync m) => t m (m a) -> t m a +sequence :: Monad m => Stream m (m a) -> Stream m a sequence = go where go m1 = mkStream $ \st yld sng stp -> let single ma = ma >>= sng - yieldk ma r = foldStreamShared st yld sng stp $ ma |: go r + yieldk ma r = foldStreamShared st yld sng stp $ ma `consM` go r in foldStream (adaptState st) yieldk single stp m1 ------------------------------------------------------------------------------- @@ -924,23 +681,26 @@ sequence = go ------------------------------------------------------------------------------- {-# INLINE intersperseM #-} -intersperseM :: (IsStream t, MonadAsync m) => m a -> t m a -> t m a +intersperseM :: Monad m => m a -> Stream m a -> Stream m a intersperseM a = prependingStart where prependingStart m1 = mkStream $ \st yld sng stp -> - let yieldk i x = foldStreamShared st yld sng stp $ return i |: go x + let yieldk i x = + foldStreamShared st yld sng stp $ return i `consM` go x in foldStream st yieldk sng stp m1 go m2 = mkStream $ \st yld sng stp -> - let single i = foldStreamShared st yld sng stp $ a |: fromPure i - yieldk i x = foldStreamShared st yld sng stp $ a |: return i |: go x + let single i = foldStreamShared st yld sng stp $ a `consM` fromPure i + yieldk i x = + foldStreamShared + st yld sng stp $ a `consM` return i `consM` go x in foldStream st yieldk single stp m2 {-# INLINE intersperse #-} -intersperse :: (IsStream t, MonadAsync m) => a -> t m a -> t m a +intersperse :: Monad m => a -> Stream m a -> Stream m a intersperse a = intersperseM (return a) {-# INLINE insertBy #-} -insertBy :: IsStream t => (a -> a -> Ordering) -> a -> t m a -> t m a +insertBy :: (a -> a -> Ordering) -> a -> Stream m a -> Stream m a insertBy cmp x = go where go m1 = mkStream $ \st yld _ _ -> @@ -958,7 +718,7 @@ insertBy cmp x = go ------------------------------------------------------------------------------ {-# INLINE deleteBy #-} -deleteBy :: IsStream t => (a -> a -> Bool) -> a -> t m a -> t m a +deleteBy :: (a -> a -> Bool) -> a -> Stream m a -> Stream m a deleteBy eq x = go where go m1 = mkStream $ \st yld sng stp -> @@ -968,20 +728,12 @@ deleteBy eq x = go else yld a (go r) in foldStream st yieldk single stp m1 ------------------------------------------------------------------------------- --- Reordering ------------------------------------------------------------------------------- - -{-# INLINE reverse #-} -reverse :: IsStream t => t m a -> t m a -reverse = foldlS (flip cons) nil - ------------------------------------------------------------------------------- -- Map and Filter ------------------------------------------------------------------------------- {-# INLINE mapMaybe #-} -mapMaybe :: IsStream t => (a -> Maybe b) -> t m a -> t m b +mapMaybe :: (a -> Maybe b) -> Stream m a -> Stream m b mapMaybe f = go where go m1 = mkStream $ \st yld sng stp -> @@ -999,7 +751,7 @@ mapMaybe f = go -- -- @since 0.1.0 {-# INLINABLE zipWith #-} -zipWith :: IsStream t => (a -> b -> c) -> t m a -> t m b -> t m c +zipWith :: (a -> b -> c) -> Stream m a -> Stream m b -> Stream m c zipWith f = go where go mx my = mkStream $ \st yld sng stp -> do @@ -1015,7 +767,8 @@ zipWith f = go -- -- @since 0.1.0 {-# INLINABLE zipWithM #-} -zipWithM :: (IsStream t, Monad m) => (a -> b -> m c) -> t m a -> t m b -> t m c +zipWithM :: Monad m => + (a -> b -> m c) -> Stream m a -> Stream m b -> Stream m c zipWithM f = go where go mx my = mkStream $ \st yld sng stp -> do @@ -1033,9 +786,8 @@ zipWithM f = go ------------------------------------------------------------------------------ {-# INLINE mergeByM #-} -mergeByM - :: (IsStream t, Monad m) - => (a -> a -> m Ordering) -> t m a -> t m a -> t m a +mergeByM :: Monad m => + (a -> a -> m Ordering) -> Stream m a -> Stream m a -> Stream m a mergeByM cmp = go where go mx my = mkStream $ \st yld sng stp -> do @@ -1058,9 +810,8 @@ mergeByM cmp = go foldStream st yieldX singleX stopX mx {-# INLINABLE mergeBy #-} -mergeBy - :: (IsStream t, Monad m) - => (a -> a -> Ordering) -> t m a -> t m a -> t m a +mergeBy :: Monad m => + (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a mergeBy cmp = mergeByM (\a b -> return $ cmp a b) ------------------------------------------------------------------------------ @@ -1068,7 +819,7 @@ mergeBy cmp = mergeByM (\a b -> return $ cmp a b) ------------------------------------------------------------------------------ {-# INLINE the #-} -the :: (Eq a, IsStream t, Monad m) => t m a -> m (Maybe a) +the :: (Eq a, Monad m) => Stream m a -> m (Maybe a) the m = do r <- uncons m case r of @@ -1091,18 +842,6 @@ _alt m1 m2 = mkStream $ \st yld sng stp -> let stop = foldStream st yld sng stp m2 in foldStream st yld sng stop m1 ------------------------------------------------------------------------------- --- MonadReader ------------------------------------------------------------------------------- - -{-# INLINABLE withLocal #-} -withLocal :: MonadReader r m => (r -> r) -> Stream m a -> Stream m a -withLocal f m = - mkStream $ \st yld sng stp -> - let single = local f . sng - yieldk a r = local f $ yld a (withLocal f r) - in foldStream st yieldk single (local f stp) m - ------------------------------------------------------------------------------ -- MonadError ------------------------------------------------------------------------------ diff --git a/src/Streamly/Internal/Data/Stream/StreamK/Type.hs b/src/Streamly/Internal/Data/Stream/StreamK/Type.hs index 23b60cbbc8..7e08e00ce5 100644 --- a/src/Streamly/Internal/Data/Stream/StreamK/Type.hs +++ b/src/Streamly/Internal/Data/Stream/StreamK/Type.hs @@ -17,26 +17,15 @@ -- module Streamly.Internal.Data.Stream.StreamK.Type ( - -- * A class for streams - IsStream (..) - , adapt - -- * The stream type - , Stream (..) + Stream (..) + , toStreamK + , fromStreamK - -- * Construction + -- * foldr/build , mkStream - , fromStopK - , fromYieldK - , consK - - -- * Elimination , foldStream , foldStreamShared - , foldl' - , foldlx' - - -- * foldr/build , foldrM , foldrS , foldrSShared @@ -45,27 +34,51 @@ module Streamly.Internal.Data.Stream.StreamK.Type , buildS , buildM , buildSM - , sharedM , augmentS , augmentSM - -- instances + -- * Construction + , fromStopK + , fromYieldK + , consK , cons , (.:) - , consMStream + , consM , consMBy + , nil + , nilM + + -- * Generation , fromEffect , fromPure + , unfoldr + , unfoldrMWith + , repeat + , repeatMWith + , replicateMWith + , fromIndicesMWith + , iterateMWith + , fromFoldable + , fromFoldableM + , mfix - , nil - , nilM + -- * Elimination + , uncons + , foldl' + , foldlx' + , drain + , null + , tail + , init + + -- * Transformation , conjoin , serial , map - , mapM + , mapMWith , mapMSerial , unShare - , concatMapBy + , concatMapWith , concatMap , bindWith , concatPairsWith @@ -73,23 +86,25 @@ module Streamly.Internal.Data.Stream.StreamK.Type , apSerial , apSerialDiscardFst , apSerialDiscardSnd + , foldlS + , reverse - , Streaming -- deprecated + -- * Reader + , withLocal ) where import Control.Monad (ap, (>=>)) +import Control.Monad.Reader.Class (MonadReader(..)) import Control.Monad.Trans.Class (MonadTrans(lift)) -import Data.Kind (Type) -#if __GLASGOW_HASKELL__ < 808 +import Data.Function (fix) +#if __GLASGOW_HASKELL__ < 804 import Data.Semigroup (Semigroup(..)) #endif -import Streamly.Internal.Control.Concurrent (MonadAsync) import Streamly.Internal.Data.SVar.Type (State, adaptState, defState) -import Prelude hiding (map, mapM, concatMap, foldr) - --- $setup --- >>> import Streamly.Prelude as Stream +import Prelude hiding + (map, mapM, concatMap, foldr, repeat, null, reverse, tail, init) +import qualified Prelude ------------------------------------------------------------------------------ -- Basic stream type @@ -124,140 +139,22 @@ newtype Stream m a = -> m r ) ------------------------------------------------------------------------------- --- Types that can behave as a Stream ------------------------------------------------------------------------------- - -infixr 5 `consM` -infixr 5 |: - --- XXX Use a different SVar based on the stream type. But we need to make sure --- that we do not lose performance due to polymorphism. --- --- | Class of types that can represent a stream of elements of some type 'a' in --- some monad 'm'. --- --- /Since: 0.2.0 ("Streamly")/ --- --- @since 0.8.0 -class -#if __GLASGOW_HASKELL__ >= 806 - ( forall m a. MonadAsync m => Semigroup (t m a) - , forall m a. MonadAsync m => Monoid (t m a) - , forall m. Monad m => Functor (t m) - , forall m. MonadAsync m => Applicative (t m) - ) => -#endif - IsStream t where - toStream :: t m a -> Stream m a - fromStream :: Stream m a -> t m a - -- | Constructs a stream by adding a monadic action at the head of an - -- existing stream. For example: - -- - -- @ - -- > toList $ getLine \`consM` getLine \`consM` nil - -- hello - -- world - -- ["hello","world"] - -- @ - -- - -- /Concurrent (do not use 'fromParallel' to construct infinite streams)/ - -- - -- @since 0.2.0 - consM :: MonadAsync m => m a -> t m a -> t m a - -- | Operator equivalent of 'consM'. We can read it as "@parallel colon@" - -- to remember that @|@ comes before ':'. - -- - -- @ - -- > toList $ getLine |: getLine |: nil - -- hello - -- world - -- ["hello","world"] - -- @ - -- - -- @ - -- let delay = threadDelay 1000000 >> print 1 - -- drain $ fromSerial $ delay |: delay |: delay |: nil - -- drain $ fromParallel $ delay |: delay |: delay |: nil - -- @ - -- - -- /Concurrent (do not use 'fromParallel' to construct infinite streams)/ - -- - -- @since 0.2.0 - (|:) :: MonadAsync m => m a -> t m a -> t m a - -- We can define (|:) just as 'consM' but it is defined explicitly for each - -- type because we want to use SPECIALIZE pragma on the definition. - --- | Same as 'IsStream'. --- --- @since 0.1.0 -{-# DEPRECATED Streaming "Please use IsStream instead." #-} -type Streaming = IsStream - -------------------------------------------------------------------------------- --- Type adapting combinators -------------------------------------------------------------------------------- - --- XXX Move/reset the State here by reconstructing the stream with cleared --- state. Can we make sure we do not do that when t1 = t2? If we do this then --- we do not need to do that explicitly using svarStyle. It would act as --- unShare when the stream type is the same. --- --- | Adapt any specific stream type to any other specific stream type. --- --- /Since: 0.1.0 ("Streamly")/ --- --- @since 0.8.0 -adapt :: (IsStream t1, IsStream t2) => t1 m a -> t2 m a -adapt = fromStream . toStream - ------------------------------------------------------------------------------- --- Building a stream ------------------------------------------------------------------------------- +{-# INLINE fromStreamK #-} +fromStreamK :: Stream m a -> Stream m a +fromStreamK = id --- XXX The State is always parameterized by "Stream" which means State is not --- different for different stream types. So we have to manually make sure that --- when converting from one stream to another we migrate the state correctly. --- This can be fixed if we use a different SVar type for different streams. --- Currently we always use "SVar Stream" and therefore a different State type --- parameterized by that stream. --- --- XXX Since t is coercible we should be able to coerce k --- mkStream k = fromStream $ MkStream $ coerce k --- --- | Build a stream from an 'SVar', a stop continuation, a singleton stream --- continuation and a yield continuation. -{-# INLINE_EARLY mkStream #-} -mkStream :: IsStream t - => (forall r. State Stream m a - -> (a -> t m a -> m r) - -> (a -> m r) - -> m r - -> m r) - -> t m a -mkStream k = fromStream $ MkStream $ \st yld sng stp -> - let yieldk a r = yld a (toStream r) - in k st yieldk sng stp - -{-# RULES "mkStream from stream" mkStream = mkStreamFromStream #-} -mkStreamFromStream :: IsStream t - => (forall r. State Stream m a - -> (a -> Stream m a -> m r) - -> (a -> m r) - -> m r - -> m r) - -> t m a -mkStreamFromStream k = fromStream $ MkStream k +{-# INLINE toStreamK #-} +toStreamK :: Stream m a -> Stream m a +toStreamK = id -{-# RULES "mkStream stream" mkStream = mkStreamStream #-} -mkStreamStream +mkStream :: (forall r. State Stream m a -> (a -> Stream m a -> m r) -> (a -> m r) -> m r -> m r) -> Stream m a -mkStreamStream = MkStream +mkStream = MkStream -- | A terminal function that has no continuation to follow. type StopK m = forall r. m r -> m r @@ -272,17 +169,17 @@ _wrapM :: Monad m => m a -> YieldK m a _wrapM m = (m >>=) -- | Make an empty stream from a stop function. -fromStopK :: IsStream t => StopK m -> t m a +fromStopK :: StopK m -> Stream m a fromStopK k = mkStream $ \_ _ _ stp -> k stp -- | Make a singleton stream from a callback function. The callback function -- calls the one-shot yield continuation to yield an element. -fromYieldK :: IsStream t => YieldK m a -> t m a +fromYieldK :: YieldK m a -> Stream m a fromYieldK k = mkStream $ \_ _ sng _ -> k sng -- | Add a yield function at the head of the stream. -consK :: IsStream t => YieldK m a -> t m a -> t m a -consK k r = mkStream $ \_ yld _ _ -> k (`yld` r) +consK :: YieldK m a -> Stream m a -> Stream m a +consK k r = mkStream $ \_ yld _ _ -> k (\x -> yld x r) -- XXX Build a stream from a repeating callback function. @@ -305,8 +202,8 @@ infixr 5 `cons` -- -- @since 0.1.0 {-# INLINE_NORMAL cons #-} -cons :: IsStream t => a -> t m a -> t m a -cons a r = mkStream $ \_ yld _ _ -> yld a r +cons :: a -> Stream m a -> Stream m a +cons a r = mkStream $ \_ yield _ _ -> yield a r infixr 5 .: @@ -319,7 +216,7 @@ infixr 5 .: -- -- @since 0.1.1 {-# INLINE (.:) #-} -(.:) :: IsStream t => a -> t m a -> t m a +(.:) :: a -> Stream m a -> Stream m a (.:) = cons -- | An empty stream. @@ -331,7 +228,7 @@ infixr 5 .: -- -- @since 0.1.0 {-# INLINE_NORMAL nil #-} -nil :: IsStream t => t m a +nil :: Stream m a nil = mkStream $ \_ _ _ stp -> stp -- | An empty stream producing a side effect. @@ -344,22 +241,32 @@ nil = mkStream $ \_ _ _ stp -> stp -- -- /Pre-release/ {-# INLINE_NORMAL nilM #-} -nilM :: (IsStream t, Monad m) => m b -> t m a +nilM :: Monad m => m b -> Stream m a nilM m = mkStream $ \_ _ _ stp -> m >> stp {-# INLINE_NORMAL fromPure #-} -fromPure :: IsStream t => a -> t m a +fromPure :: a -> Stream m a fromPure a = mkStream $ \_ _ single _ -> single a {-# INLINE_NORMAL fromEffect #-} -fromEffect :: (Monad m, IsStream t) => m a -> t m a -fromEffect m = fromStream $ mkStream $ \_ _ single _ -> m >>= single +fromEffect :: Monad m => m a -> Stream m a +fromEffect m = mkStream $ \_ _ single _ -> m >>= single + +infixr 5 `consM` + +-- NOTE: specializing the function outside the instance definition seems to +-- improve performance quite a bit at times, even if we have the same +-- SPECIALIZE in the instance definition. +{-# INLINE consM #-} +{-# SPECIALIZE consM :: IO a -> Stream IO a -> Stream IO a #-} +consM :: (Monad m) => m a -> Stream m a -> Stream m a +consM m r = MkStream $ \_ yld _ _ -> m >>= \a -> yld a r -- XXX specialize to IO? {-# INLINE consMBy #-} -consMBy :: (IsStream t, MonadAsync m) => (t m a -> t m a -> t m a) - -> m a -> t m a -> t m a -consMBy f m r = fromStream (fromEffect m) `f` r +consMBy :: Monad m => + (Stream m a -> Stream m a -> Stream m a) -> m a -> Stream m a -> Stream m a +consMBy f m r = fromEffect m `f` r ------------------------------------------------------------------------------ -- Folding a stream @@ -370,78 +277,27 @@ consMBy f m r = fromStream (fromEffect m) `f` r -- SVar passed via the State. {-# INLINE_EARLY foldStreamShared #-} foldStreamShared - :: IsStream t - => State Stream m a - -> (a -> t m a -> m r) - -> (a -> m r) - -> m r - -> t m a - -> m r -foldStreamShared st yld sng stp m = - let yieldk a x = yld a (fromStream x) - MkStream k = toStream m - in k st yieldk sng stp - --- XXX write a similar rule for foldStream as well? -{-# RULES "foldStreamShared from stream" - foldStreamShared = foldStreamSharedStream #-} -foldStreamSharedStream :: State Stream m a -> (a -> Stream m a -> m r) -> (a -> m r) -> m r -> Stream m a -> m r -foldStreamSharedStream st yld sng stp m = - let MkStream k = toStream m - in k st yld sng stp +foldStreamShared s yield single stop (MkStream k) = k s yield single stop -- | Fold a stream by providing a State, stop continuation, a singleton -- continuation and a yield continuation. The stream will not use the SVar -- passed via State. {-# INLINE foldStream #-} foldStream - :: IsStream t - => State Stream m a - -> (a -> t m a -> m r) + :: State Stream m a + -> (a -> Stream m a -> m r) -> (a -> m r) -> m r - -> t m a + -> Stream m a -> m r -foldStream st yld sng stp m = - let yieldk a x = yld a (fromStream x) - MkStream k = toStream m - in k (adaptState st) yieldk sng stp - -------------------------------------------------------------------------------- --- Instances -------------------------------------------------------------------------------- - --- NOTE: specializing the function outside the instance definition seems to --- improve performance quite a bit at times, even if we have the same --- SPECIALIZE in the instance definition. -{-# INLINE consMStream #-} -{-# SPECIALIZE consMStream :: IO a -> Stream IO a -> Stream IO a #-} -consMStream :: (Monad m) => m a -> Stream m a -> Stream m a -consMStream m r = MkStream $ \_ yld _ _ -> m >>= \a -> yld a r - -------------------------------------------------------------------------------- --- IsStream Stream -------------------------------------------------------------------------------- - -instance IsStream Stream where - toStream = id - fromStream = id - - {-# INLINE consM #-} - {-# SPECIALIZE consM :: IO a -> Stream IO a -> Stream IO a #-} - consM :: Monad m => m a -> Stream m a -> Stream m a - consM = consMStream - - {-# INLINE (|:) #-} - {-# SPECIALIZE (|:) :: IO a -> Stream IO a -> Stream IO a #-} - (|:) :: Monad m => m a -> Stream m a -> Stream m a - (|:) = consMStream +foldStream s yield single stop (MkStream k) = + k (adaptState s) yield single stop ------------------------------------------------------------------------------- -- foldr/build fusion @@ -454,15 +310,18 @@ instance IsStream Stream where -- reconstruct using a shared state (SVar) or without sharing the state. -- {-# INLINE foldrSWith #-} -foldrSWith :: IsStream t - => (forall r. State Stream m b - -> (b -> t m b -> m r) +foldrSWith :: + (forall r. State Stream m b + -> (b -> Stream m b -> m r) -> (b -> m r) -> m r - -> t m b + -> Stream m b -> m r) - -> (a -> t m b -> t m b) -> t m b -> t m a -> t m b -foldrSWith f step final = go + -> (a -> Stream m b -> Stream m b) + -> Stream m b + -> Stream m a + -> Stream m b +foldrSWith f step final m = go m where go m1 = mkStream $ \st yld sng stp -> let run x = f st yld sng stp x @@ -480,7 +339,11 @@ foldrSWith f step final = go -- | Fold sharing the SVar state within the reconstructed stream {-# INLINE_NORMAL foldrSShared #-} -foldrSShared :: IsStream t => (a -> t m b -> t m b) -> t m b -> t m a -> t m b +foldrSShared :: + (a -> Stream m b -> Stream m b) + -> Stream m b + -> Stream m a + -> Stream m b foldrSShared = foldrSWith foldStreamShared -- XXX consM is a typeclass method, therefore rewritten already. Instead maybe @@ -495,7 +358,11 @@ foldrSShared = foldrSWith foldStreamShared -- | Lazy right associative fold to a stream. {-# INLINE_NORMAL foldrS #-} -foldrS :: IsStream t => (a -> t m b -> t m b) -> t m b -> t m a -> t m b +foldrS :: + (a -> Stream m b -> Stream m b) + -> Stream m b + -> Stream m a + -> Stream m b foldrS = foldrSWith foldStream {-# RULES "foldrS/id" foldrS cons nil = \x -> x #-} @@ -512,15 +379,18 @@ foldrS = foldrSWith foldStream ------------------------------------------------------------------------------- {-# INLINE foldrSMWith #-} -foldrSMWith :: (IsStream t, Monad m) +foldrSMWith :: Monad m => (forall r. State Stream m b - -> (b -> t m b -> m r) + -> (b -> Stream m b -> m r) -> (b -> m r) -> m r - -> t m b + -> Stream m b -> m r) - -> (m a -> t m b -> t m b) -> t m b -> t m a -> t m b -foldrSMWith f step final = go + -> (m a -> Stream m b -> Stream m b) + -> Stream m b + -> Stream m a + -> Stream m b +foldrSMWith f step final m = go m where go m1 = mkStream $ \st yld sng stp -> let run x = f st yld sng stp x @@ -530,8 +400,11 @@ foldrSMWith f step final = go in foldStream (adaptState st) yieldk single stop m1 {-# INLINE_NORMAL foldrSM #-} -foldrSM :: (IsStream t, Monad m) - => (m a -> t m b -> t m b) -> t m b -> t m a -> t m b +foldrSM :: Monad m + => (m a -> Stream m b -> Stream m b) + -> Stream m b + -> Stream m a + -> Stream m b foldrSM = foldrSMWith foldStream -- {-# RULES "foldrSM/id" foldrSM consM nil = \x -> x #-} @@ -542,8 +415,11 @@ foldrSM = foldrSMWith foldStream -- Like foldrSM but sharing the SVar state within the recostructed stream. {-# INLINE_NORMAL foldrSMShared #-} -foldrSMShared :: (IsStream t, Monad m) - => (m a -> t m b -> t m b) -> t m b -> t m a -> t m b +foldrSMShared :: Monad m + => (m a -> Stream m b -> Stream m b) + -> Stream m b + -> Stream m a + -> Stream m b foldrSMShared = foldrSMWith foldStreamShared -- {-# RULES "foldrSM/id" foldrSM consM nil = \x -> x #-} @@ -559,7 +435,7 @@ foldrSMShared = foldrSMWith foldStreamShared ------------------------------------------------------------------------------- {-# INLINE_NORMAL build #-} -build :: IsStream t => forall a. (forall b. (a -> b -> b) -> b -> b) -> t m a +build :: forall m a. (forall b. (a -> b -> b) -> b -> b) -> Stream m a build g = g cons nil {-# RULES "foldrM/build" @@ -584,37 +460,46 @@ build g = g cons nil -- build a stream by applying cons and nil to a build function {-# INLINE_NORMAL buildS #-} -buildS :: IsStream t => ((a -> t m a -> t m a) -> t m a -> t m a) -> t m a +buildS :: + ((a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a) + -> Stream m a buildS g = g cons nil {-# RULES "foldrS/buildS" - forall k z (g :: (a -> t m a -> t m a) -> t m a -> t m a). + forall k z + (g :: (a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a). foldrS k z (buildS g) = g k z #-} {-# RULES "foldrS/cons/buildS" - forall k z x (g :: (a -> t m a -> t m a) -> t m a -> t m a). + forall k z x + (g :: (a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a). foldrS k z (x `cons` buildS g) = k x (g k z) #-} {-# RULES "foldrSShared/buildS" - forall k z (g :: (a -> t m a -> t m a) -> t m a -> t m a). + forall k z + (g :: (a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a). foldrSShared k z (buildS g) = g k z #-} {-# RULES "foldrSShared/cons/buildS" - forall k z x (g :: (a -> t m a -> t m a) -> t m a -> t m a). + forall k z x + (g :: (a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a). foldrSShared k z (x `cons` buildS g) = k x (g k z) #-} -- build a stream by applying consM and nil to a build function {-# INLINE_NORMAL buildSM #-} -buildSM :: (IsStream t, MonadAsync m) - => ((m a -> t m a -> t m a) -> t m a -> t m a) -> t m a +buildSM :: Monad m + => ((m a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a) + -> Stream m a buildSM g = g consM nil {-# RULES "foldrSM/buildSM" - forall k z (g :: (m a -> t m a -> t m a) -> t m a -> t m a). + forall k z + (g :: (m a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a). foldrSM k z (buildSM g) = g k z #-} {-# RULES "foldrSMShared/buildSM" - forall k z (g :: (m a -> t m a -> t m a) -> t m a -> t m a). + forall k z + (g :: (m a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a). foldrSMShared k z (buildSM g) = g k z #-} -- Disabled because this may not fire as consM is a class Op @@ -629,71 +514,79 @@ buildSM g = g consM nil -- Build using monadic build functions (continuations) instead of -- reconstructing a stream. {-# INLINE_NORMAL buildM #-} -buildM :: (IsStream t, MonadAsync m) - => (forall r. (a -> t m a -> m r) +buildM :: Monad m + => (forall r. (a -> Stream m a -> m r) -> (a -> m r) -> m r -> m r ) - -> t m a + -> Stream m a buildM g = mkStream $ \st yld sng stp -> g (\a r -> foldStream st yld sng stp (return a `consM` r)) sng stp -- | Like 'buildM' but shares the SVar state across computations. -{-# INLINE_NORMAL sharedM #-} -sharedM :: (IsStream t, MonadAsync m) - => (forall r. (a -> t m a -> m r) +{-# INLINE_NORMAL sharedMWith #-} +sharedMWith :: Monad m + => (m a -> Stream m a -> Stream m a) + -> (forall r. (a -> Stream m a -> m r) -> (a -> m r) -> m r -> m r ) - -> t m a -sharedM g = mkStream $ \st yld sng stp -> - g (\a r -> foldStreamShared st yld sng stp (return a `consM` r)) sng stp + -> Stream m a +sharedMWith cns g = mkStream $ \st yld sng stp -> + g (\a r -> foldStreamShared st yld sng stp (return a `cns` r)) sng stp ------------------------------------------------------------------------------- -- augment ------------------------------------------------------------------------------- {-# INLINE_NORMAL augmentS #-} -augmentS :: IsStream t - => ((a -> t m a -> t m a) -> t m a -> t m a) -> t m a -> t m a -augmentS g = g cons +augmentS :: + ((a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a) + -> Stream m a + -> Stream m a +augmentS g xs = g cons xs {-# RULES "augmentS/nil" - forall (g :: (a -> t m a -> t m a) -> t m a -> t m a). + forall (g :: (a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a). augmentS g nil = buildS g #-} {-# RULES "foldrS/augmentS" - forall k z xs (g :: (a -> t m a -> t m a) -> t m a -> t m a). + forall k z xs + (g :: (a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a). foldrS k z (augmentS g xs) = g k (foldrS k z xs) #-} {-# RULES "augmentS/buildS" - forall (g :: (a -> t m a -> t m a) -> t m a -> t m a) - (h :: (a -> t m a -> t m a) -> t m a -> t m a). + forall (g :: (a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a) + (h :: (a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a). augmentS g (buildS h) = buildS (\c n -> g c (h c n)) #-} {-# INLINE_NORMAL augmentSM #-} -augmentSM :: (IsStream t, MonadAsync m) - => ((m a -> t m a -> t m a) -> t m a -> t m a) -> t m a -> t m a -augmentSM g = g consM +augmentSM :: Monad m => + ((m a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a) + -> Stream m a -> Stream m a +augmentSM g xs = g consM xs {-# RULES "augmentSM/nil" - forall (g :: (m a -> t m a -> t m a) -> t m a -> t m a). + forall + (g :: (m a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a). augmentSM g nil = buildSM g #-} {-# RULES "foldrSM/augmentSM" - forall k z xs (g :: (m a -> t m a -> t m a) -> t m a -> t m a). + forall k z xs + (g :: (m a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a). foldrSM k z (augmentSM g xs) = g k (foldrSM k z xs) #-} {-# RULES "augmentSM/buildSM" - forall (g :: (m a -> t m a -> t m a) -> t m a -> t m a) - (h :: (m a -> t m a -> t m a) -> t m a -> t m a). + forall + (g :: (m a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a) + (h :: (m a -> Stream m a -> Stream m a) -> Stream m a -> Stream m a). augmentSM g (buildSM h) = buildSM (\c n -> g c (h c n)) #-} @@ -703,8 +596,8 @@ augmentSM g = g consM -- | Lazy right fold with a monadic step function. {-# INLINE_NORMAL foldrM #-} -foldrM :: IsStream t => (a -> m b -> m b) -> m b -> t m a -> m b -foldrM step acc = go +foldrM :: (a -> m b -> m b) -> m b -> Stream m a -> m b +foldrM step acc m = go m where go m1 = let stop = acc @@ -715,14 +608,14 @@ foldrM step acc = go {-# INLINE_NORMAL foldrMKWith #-} foldrMKWith :: (State Stream m a - -> (a -> t m a -> m b) + -> (a -> Stream m a -> m b) -> (a -> m b) -> m b - -> t m a + -> Stream m a -> m b) -> (a -> m b -> m b) -> m b - -> ((a -> t m a -> m b) -> (a -> m b) -> m b -> m b) + -> ((a -> Stream m a -> m b) -> (a -> m b) -> m b -> m b) -> m b foldrMKWith f step acc = go where @@ -742,7 +635,7 @@ foldrMKWith f step acc = go -- XXX in which case will foldrM/buildM fusion be useful? {-# RULES "foldrM/buildM" forall step acc (g :: (forall r. - (a -> t m a -> m r) + (a -> Stream m a -> m r) -> (a -> m r) -> m r -> m r @@ -750,15 +643,17 @@ foldrMKWith f step acc = go foldrM step acc (buildM g) = foldrMKWith foldStream step acc g #-} +{- {-# RULES "foldrM/sharedM" forall step acc (g :: (forall r. - (a -> t m a -> m r) + (a -> Stream m a -> m r) -> (a -> m r) -> m r -> m r )). foldrM step acc (sharedM g) = foldrMKWith foldStreamShared step acc g #-} +-} ------------------------------------------------------------------------------ -- Left fold @@ -771,12 +666,12 @@ foldrMKWith f step acc = go -- -- Note that the accumulator is always evaluated including the initial value. {-# INLINE foldlx' #-} -foldlx' :: forall t m a b x. (IsStream t, Monad m) - => (x -> a -> x) -> x -> (x -> b) -> t m a -> m b +foldlx' :: forall m a b x. Monad m + => (x -> a -> x) -> x -> (x -> b) -> Stream m a -> m b foldlx' step begin done m = get $ go m begin where {-# NOINLINE get #-} - get :: t m x -> m b + get :: Stream m x -> m b get m1 = -- XXX we are not strictly evaluating the accumulator here. Is this -- okay? @@ -788,7 +683,7 @@ foldlx' step begin done m = get $ go m begin -- however that is more expensive because of unnecessary recursion -- that cannot be tail call optimized. Unfolding recursion explicitly via -- continuations is much more efficient. - go :: t m a -> x -> t m x + go :: Stream m a -> x -> Stream m x go m1 !acc = mkStream $ \_ yld sng _ -> let stop = sng acc single a = sng $ step acc a @@ -799,9 +694,44 @@ foldlx' step begin done m = get $ go m begin -- | Strict left associative fold. {-# INLINE foldl' #-} -foldl' :: (IsStream t, Monad m) => (b -> a -> b) -> b -> t m a -> m b +foldl' :: Monad m => (b -> a -> b) -> b -> Stream m a -> m b foldl' step begin = foldlx' step begin id +------------------------------------------------------------------------------ +-- Specialized folds +------------------------------------------------------------------------------ + +-- XXX use foldrM to implement folds where possible +-- XXX This (commented) definition of drain and mapM_ perform much better on +-- some benchmarks but worse on others. Need to investigate why, may there is +-- an optimization opportunity that we can exploit. +-- drain = foldrM (\_ xs -> return () >> xs) (return ()) + +-- | +-- > drain = foldl' (\_ _ -> ()) () +-- > drain = mapM_ (\_ -> return ()) +{-# INLINE drain #-} +drain :: Monad m => Stream m a -> m () +drain = foldrM (\_ xs -> xs) (return ()) +{- +drain = go + where + go m1 = + let stop = return () + single _ = return () + yieldk _ r = go r + in foldStream defState yieldk single stop m1 +-} + +{-# INLINE null #-} +null :: Monad m => Stream m a -> m Bool +-- null = foldrM (\_ _ -> return True) (return False) +null m = + let stop = return True + single _ = return False + yieldk _ _ = return False + in foldStream defState yieldk single stop m + ------------------------------------------------------------------------------ -- Semigroup ------------------------------------------------------------------------------ @@ -811,19 +741,8 @@ infixr 6 `serial` -- | Appends two streams sequentially, yielding all elements from the first -- stream, and then all elements from the second stream. -- --- >>> import Streamly.Prelude (serial) --- >>> stream1 = Stream.fromList [1,2] --- >>> stream2 = Stream.fromList [3,4] --- >>> Stream.toList $ stream1 `serial` stream2 --- [1,2,3,4] --- --- This operation can be used to fold an infinite lazy container of streams. --- --- /Since: 0.2.0 ("Streamly")/ --- --- @since 0.8.0 {-# INLINE serial #-} -serial :: IsStream t => t m a -> t m a -> t m a +serial :: Stream m a -> Stream m a -> Stream m a -- XXX This doubles the time of toNullAp benchmark, may not be fusing properly -- serial xs ys = augmentS (\c n -> foldrS c n xs) ys serial m1 m2 = go m1 @@ -836,7 +755,7 @@ serial m1 m2 = go m1 -- join/merge/append streams depending on consM {-# INLINE conjoin #-} -conjoin :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a +conjoin :: Monad m => Stream m a -> Stream m a -> Stream m a conjoin xs = augmentSM (\c n -> foldrSM c n xs) instance Semigroup (Stream m a) where @@ -856,10 +775,13 @@ instance Monoid (Stream m a) where -- Note eta expanded {-# INLINE_LATE mapFB #-} -mapFB :: forall (t :: (Type -> Type) -> Type -> Type) b m a. - (b -> t m b -> t m b) -> (a -> b) -> a -> t m b -> t m b -mapFB c f x ys = c (f x) ys -#undef Type +mapFB :: forall b m a. + (b -> Stream m b -> Stream m b) + -> (a -> b) + -> a + -> Stream m b + -> Stream m b +mapFB c f = \x ys -> c (f x) ys {-# RULES "mapFB/mapFB" forall c f g. mapFB (mapFB c f) g = mapFB c (f . g) @@ -867,7 +789,7 @@ mapFB c f x ys = c (f x) ys #-} {-# INLINE map #-} -map :: IsStream t => (a -> b) -> t m a -> t m b +map :: (a -> b) -> Stream m a -> Stream m b map f xs = buildS (\c n -> foldrS (mapFB c f) n xs) -- XXX This definition might potentially be more efficient, but the cost in the @@ -898,21 +820,6 @@ mapMFB c f x = c (x >>= f) "mapMFB/return" forall c. mapMFB c (\x -> return x) = c -} --- Be careful when modifying this, this uses a consM (|:) deliberately to allow --- other stream types to overload it. -{-# INLINE mapM #-} -mapM :: (IsStream t, MonadAsync m) => (a -> m b) -> t m a -> t m b -mapM f = foldrSShared (\x xs -> f x `consM` xs) nil --- See note under map definition above. -{- -mapM f m = go m - where - go m1 = mkStream $ \st yld sng stp -> - let single a = f a >>= sng - yieldk a r = foldStreamShared st yld sng stp $ f a |: go r - in foldStream (adaptState st) yieldk single stp m1 - -} - -- This is experimental serial version supporting fusion. -- -- XXX what if we do not want to fuse two concurrent mapMs? @@ -921,9 +828,27 @@ mapM f m = go m -- XXX fusion would be easier for monomoprhic stream types. -- {-# RULES "mapM serial" mapM = mapMSerial #-} {-# INLINE mapMSerial #-} -mapMSerial :: MonadAsync m => (a -> m b) -> Stream m a -> Stream m b +mapMSerial :: Monad m => (a -> m b) -> Stream m a -> Stream m b mapMSerial f xs = buildSM (\c n -> foldrSMShared (mapMFB c f) n xs) +{-# INLINE mapMWith #-} +mapMWith :: + (m b -> Stream m b -> Stream m b) + -> (a -> m b) + -> Stream m a + -> Stream m b +mapMWith cns f = foldrSShared (\x xs -> f x `cns` xs) nil + +{- +-- See note under map definition above. +mapMWith cns f = go + where + go m1 = mkStream $ \st yld sng stp -> + let single a = f a >>= sng + yieldk a r = foldStreamShared st yld sng stp $ f a `cns` go r + in foldStream (adaptState st) yieldk single stp m1 +-} + -- XXX in fact use the Stream type everywhere and only use polymorphism in the -- high level modules/prelude. instance Monad m => Functor (Stream m) where @@ -943,18 +868,17 @@ instance MonadTrans Stream where -- | Detach a stream from an SVar {-# INLINE unShare #-} -unShare :: IsStream t => t m a -> t m a +unShare :: Stream m a -> Stream m a unShare x = mkStream $ \st yld sng stp -> foldStream st yld sng stp x -- XXX the function stream and value stream can run in parallel {-# INLINE apWith #-} -apWith - :: IsStream t - => (t m b -> t m b -> t m b) - -> t m (a -> b) - -> t m a - -> t m b +apWith :: + (Stream m b -> Stream m b -> Stream m b) + -> Stream m (a -> b) + -> Stream m a + -> Stream m b apWith par fstream stream = go1 fstream where @@ -973,11 +897,10 @@ apWith par fstream stream = go1 fstream in foldStream (adaptState st) yieldk single stp m {-# INLINE apSerial #-} -apSerial - :: IsStream t - => t m (a -> b) - -> t m a - -> t m b +apSerial :: + Stream m (a -> b) + -> Stream m a + -> Stream m b apSerial fstream stream = go1 fstream where @@ -1004,11 +927,10 @@ apSerial fstream stream = go1 fstream in foldStream (adaptState st) yieldk single stp m {-# INLINE apSerialDiscardFst #-} -apSerialDiscardFst - :: IsStream t - => t m a - -> t m b - -> t m b +apSerialDiscardFst :: + Stream m a + -> Stream m b + -> Stream m b apSerialDiscardFst fstream stream = go1 fstream where @@ -1029,11 +951,10 @@ apSerialDiscardFst fstream stream = go1 fstream in foldStream st yieldk single stop m {-# INLINE apSerialDiscardSnd #-} -apSerialDiscardSnd - :: IsStream t - => t m a - -> t m b - -> t m a +apSerialDiscardSnd :: + Stream m a + -> Stream m b + -> Stream m a apSerialDiscardSnd fstream stream = go1 fstream where @@ -1059,17 +980,16 @@ apSerialDiscardSnd fstream stream = go1 fstream yieldk _ r = yld f (go3 f r) in foldStream (adaptState st) yieldk single stp m --- XXX This is just concatMapBy with arguments flipped. We need to keep this +-- XXX This is just concatMapWith with arguments flipped. We need to keep this -- instead of using a concatMap style definition because the bind -- implementation in Async and WAsync streams show significant perf degradation -- if the argument order is changed. {-# INLINE bindWith #-} -bindWith - :: IsStream t - => (t m b -> t m b -> t m b) - -> t m a - -> (a -> t m b) - -> t m b +bindWith :: + (Stream m b -> Stream m b -> Stream m b) + -> Stream m a + -> (a -> Stream m b) + -> Stream m b bindWith par m1 f = go m1 where go m = @@ -1094,21 +1014,18 @@ bindWith par m1 f = go m1 -- function. -- -- @since 0.7.0 -{-# INLINE concatMapBy #-} -concatMapBy - :: IsStream t - => (t m b -> t m b -> t m b) - -> (a -> t m b) - -> t m a - -> t m b -concatMapBy par f xs = bindWith par xs f +{-# INLINE concatMapWith #-} +concatMapWith + :: + (Stream m b -> Stream m b -> Stream m b) + -> (a -> Stream m b) + -> Stream m a + -> Stream m b +concatMapWith par f xs = bindWith par xs f {-# INLINE concatMap #-} -concatMap :: IsStream t => (a -> t m b) -> t m a -> t m b -concatMap f m = fromStream $ - concatMapBy serial - (adapt . toStream . f) - (adapt $ toStream m) +concatMap :: (a -> Stream m b) -> Stream m a -> Stream m b +concatMap = concatMapWith serial {- -- Fused version. @@ -1133,11 +1050,11 @@ concatMap_ f xs = buildS -- {-# INLINE concatPairsWith #-} concatPairsWith - :: IsStream t - => (t m b -> t m b -> t m b) - -> (a -> t m b) - -> t m a - -> t m b + :: + (Stream m b -> Stream m b -> Stream m b) + -> (a -> Stream m b) + -> Stream m a + -> Stream m b concatPairsWith combine f = go Nothing where @@ -1195,3 +1112,194 @@ concatUnfoldr :: IsStream t => (b -> t m (Maybe (a, b))) -> t m b -> t m a concatUnfoldr = undefined -} + +------------------------------------------------------------------------------ +-- MonadReader +------------------------------------------------------------------------------ + +{-# INLINABLE withLocal #-} +withLocal :: MonadReader r m => (r -> r) -> Stream m a -> Stream m a +withLocal f m = + mkStream $ \st yld sng stp -> + let single = local f . sng + yieldk a r = local f $ yld a (withLocal f r) + in foldStream st yieldk single (local f stp) m + +------------------------------------------------------------------------------- +-- Generation +------------------------------------------------------------------------------- + +{-# INLINE unfoldr #-} +unfoldr :: (b -> Maybe (a, b)) -> b -> Stream m a +unfoldr next s0 = build $ \yld stp -> + let go s = + case next s of + Just (a, b) -> yld a (go b) + Nothing -> stp + in go s0 + +{-# INLINE unfoldrMWith #-} +unfoldrMWith :: Monad m => + (m a -> Stream m a -> Stream m a) + -> (b -> m (Maybe (a, b))) + -> b + -> Stream m a +unfoldrMWith cns step = go + + where + + go s = sharedMWith cns $ \yld _ stp -> do + r <- step s + case r of + Just (a, b) -> yld a (go b) + Nothing -> stp + +-- | Generate an infinite stream by repeating a pure value. +-- +-- /Pre-release/ +{-# INLINE repeat #-} +repeat :: a -> Stream m a +repeat a = let x = cons a x in x + +-- | Like 'repeatM' but takes a stream 'cons' operation to combine the actions +-- in a stream specific manner. A serial cons would repeat the values serially +-- while an async cons would repeat concurrently. +-- +-- /Pre-release/ +repeatMWith :: (m a -> t m a -> t m a) -> m a -> t m a +repeatMWith cns = go + + where + + go m = m `cns` go m + +{-# INLINE replicateMWith #-} +replicateMWith :: (m a -> Stream m a -> Stream m a) -> Int -> m a -> Stream m a +replicateMWith cns n m = go n + + where + + go cnt = if cnt <= 0 then nil else m `cns` go (cnt - 1) + +{-# INLINE fromIndicesMWith #-} +fromIndicesMWith :: + (m a -> Stream m a -> Stream m a) -> (Int -> m a) -> Stream m a +fromIndicesMWith cns gen = go 0 + + where + + go i = mkStream $ \st stp sng yld -> do + foldStreamShared st stp sng yld (gen i `cns` go (i + 1)) + +{-# INLINE iterateMWith #-} +iterateMWith :: Monad m => + (m a -> Stream m a -> Stream m a) -> (a -> m a) -> m a -> Stream m a +iterateMWith cns step = go + + where + + go s = mkStream $ \st stp sng yld -> do + !next <- s + foldStreamShared st stp sng yld (return next `cns` go (step next)) + +{-# INLINE headPartial #-} +headPartial :: Monad m => Stream m a -> m a +headPartial = foldrM (\x _ -> return x) (error "head of nil") + +{-# INLINE tailPartial #-} +tailPartial :: Stream m a -> Stream m a +tailPartial m = mkStream $ \st yld sng stp -> + let stop = error "tail of nil" + single _ = stp + yieldk _ r = foldStream st yld sng stp r + in foldStream st yieldk single stop m + +{-# INLINE mfix #-} +mfix :: Monad m => (m a -> Stream m a) -> Stream m a +mfix f = mkStream $ \st yld sng stp -> + let single a = foldStream st yld sng stp $ a `cons` ys + yieldk a _ = foldStream st yld sng stp $ a `cons` ys + in foldStream st yieldk single stp xs + + where + + -- fix the head element of the stream + xs = fix (f . headPartial) + + -- now fix the tail recursively + ys = mfix (tailPartial . f) + +------------------------------------------------------------------------------- +-- Conversions +------------------------------------------------------------------------------- + +-- | +-- @ +-- fromFoldable = 'Prelude.foldr' 'cons' 'nil' +-- @ +-- +-- Construct a stream from a 'Foldable' containing pure values: +-- +-- @since 0.2.0 +{-# INLINE fromFoldable #-} +fromFoldable :: Foldable f => f a -> Stream m a +fromFoldable = Prelude.foldr cons nil + +{-# INLINE fromFoldableM #-} +fromFoldableM :: (Foldable f, Monad m) => f (m a) -> Stream m a +fromFoldableM = Prelude.foldr consM nil + +------------------------------------------------------------------------------- +-- Deconstruction +------------------------------------------------------------------------------- + +{-# INLINE uncons #-} +uncons :: Monad m => Stream m a -> m (Maybe (a, Stream m a)) +uncons m = + let stop = return Nothing + single a = return (Just (a, nil)) + yieldk a r = return (Just (a, r)) + in foldStream defState yieldk single stop m + +{-# INLINE tail #-} +tail :: Monad m => Stream m a -> m (Maybe (Stream m a)) +tail = + let stop = return Nothing + single _ = return $ Just nil + yieldk _ r = return $ Just r + in foldStream defState yieldk single stop + +{-# INLINE init #-} +init :: Monad m => Stream m a -> m (Maybe (Stream m a)) +init = go1 + where + go1 m1 = do + r <- uncons m1 + case r of + Nothing -> return Nothing + Just (h, t) -> return . Just $ go h t + go p m1 = mkStream $ \_ yld sng stp -> + let single _ = sng p + yieldk a x = yld p $ go a x + in foldStream defState yieldk single stp m1 + +------------------------------------------------------------------------------ +-- Reordering +------------------------------------------------------------------------------ + +-- | Lazy left fold to a stream. +{-# INLINE foldlS #-} +foldlS :: + (Stream m b -> a -> Stream m b) -> Stream m b -> Stream m a -> Stream m b +foldlS step = go + where + go acc rest = mkStream $ \st yld sng stp -> + let run x = foldStream st yld sng stp x + stop = run acc + single a = run $ step acc a + yieldk a r = run $ go (step acc a) r + in foldStream (adaptState st) yieldk single stop rest + +{-# INLINE reverse #-} +reverse :: Stream m a -> Stream m a +reverse = foldlS (flip cons) nil diff --git a/src/Streamly/Internal/Data/Stream/Zip.hs b/src/Streamly/Internal/Data/Stream/Zip.hs index 3498ab1f39..09d73a7c25 100644 --- a/src/Streamly/Internal/Data/Stream/Zip.hs +++ b/src/Streamly/Internal/Data/Stream/Zip.hs @@ -15,23 +15,20 @@ -- module Streamly.Internal.Data.Stream.Zip ( - ZipSerialM + ZipSerialM (..) , ZipSerial - , fromZipSerial + , consMZip + , zipWithK + , zipWithMK - , ZipAsyncM + , ZipAsyncM(..) , ZipAsync - , fromZipAsync - - , zipWith - , zipWithM - , zipAsyncWith - , zipAsyncWithM + , consMZipAsync + , zipAsyncWithK + , zipAsyncWithMK -- * Deprecated , ZipStream - , zipping - , zippingAsync ) where @@ -51,23 +48,18 @@ import GHC.Exts (IsList(..), IsString(..)) import Text.Read ( Lexeme(Ident), lexP, parens, prec, readPrec, readListPrec , readListPrecDefault) -import Streamly.Internal.Control.Concurrent (MonadAsync) import Streamly.Internal.BaseCompat ((#.), errorWithoutStackTrace, oneShot) -import Streamly.Internal.Data.Stream.StreamK (IsStream(..), Stream) +import Streamly.Internal.Control.Concurrent (MonadAsync) import Streamly.Internal.Data.Maybe.Strict (Maybe'(..), toMaybe) +import Streamly.Internal.Data.Stream.Serial (SerialT(..)) +import Streamly.Internal.Data.Stream.StreamK.Type (Stream) import qualified Streamly.Internal.Data.Stream.Parallel as Par import qualified Streamly.Internal.Data.Stream.Prelude as P - (cmpBy, eqBy, foldl', foldr, fromList, toList, fromStreamS, toStreamS) -import qualified Streamly.Internal.Data.Stream.StreamK as K (repeat) + (cmpBy, eqBy, foldl', foldr, fromList, toList) import qualified Streamly.Internal.Data.Stream.StreamK.Type as K -import qualified Streamly.Internal.Data.Stream.StreamD as D (zipWithM) -import qualified Streamly.Internal.Data.Stream.StreamD.Type as D -#ifdef USE_STREAMK_ONLY -import qualified Streamly.Internal.Data.Stream.StreamK as S (zipWith, zipWithM) -#else -import qualified Streamly.Internal.Data.Stream.StreamD as S (zipWith, zipWithM) -#endif +import qualified Streamly.Internal.Data.Stream.StreamD as D +import qualified Streamly.Internal.Data.Stream.Serial as Serial import Prelude hiding (map, repeat, zipWith, errorWithoutStackTrace) @@ -83,29 +75,16 @@ import Prelude hiding (map, repeat, zipWith, errorWithoutStackTrace) -- return n -- IO Int -- :} --- | Like 'zipWith' but using a monadic zipping function. --- --- @since 0.4.0 -{-# INLINABLE zipWithM #-} -zipWithM :: (IsStream t, Monad m) => (a -> b -> m c) -> t m a -> t m b -> t m c -zipWithM f m1 m2 = P.fromStreamS $ S.zipWithM f (P.toStreamS m1) (P.toStreamS m2) +{-# INLINE zipWithMK #-} +zipWithMK :: Monad m => + (a -> b -> m c) -> Stream m a -> Stream m b -> Stream m c +zipWithMK f m1 m2 = + D.toStreamK $ D.zipWithM f (D.fromStreamK m1) (D.fromStreamK m2) --- | Stream @a@ is evaluated first, followed by stream @b@, the resulting --- elements @a@ and @b@ are then zipped using the supplied zip function and the --- result @c@ is yielded to the consumer. --- --- If stream @a@ or stream @b@ ends, the zipped stream ends. If stream @b@ ends --- first, the element @a@ from previous evaluation of stream @a@ is discarded. --- --- @ --- > S.toList $ S.zipWith (+) (S.fromList [1,2,3]) (S.fromList [4,5,6]) --- [5,7,9] --- @ --- --- @since 0.1.0 -{-# INLINABLE zipWith #-} -zipWith :: (IsStream t, Monad m) => (a -> b -> c) -> t m a -> t m b -> t m c -zipWith f m1 m2 = P.fromStreamS $ S.zipWith f (P.toStreamS m1) (P.toStreamS m2) +{-# INLINE zipWithK #-} +zipWithK :: Monad m + => (a -> b -> c) -> Stream m a -> Stream m b -> Stream m c +zipWithK f = zipWithMK (\a b -> return (f a b)) ------------------------------------------------------------------------------ -- Parallel Zipping @@ -114,12 +93,12 @@ zipWith f m1 m2 = P.fromStreamS $ S.zipWith f (P.toStreamS m1) (P.toStreamS m2) -- | Like 'zipAsyncWith' but with a monadic zipping function. -- -- @since 0.4.0 -{-# INLINE zipAsyncWithM #-} -zipAsyncWithM :: (IsStream t, MonadAsync m) - => (a -> b -> m c) -> t m a -> t m b -> t m c -zipAsyncWithM f m1 m2 = D.fromStreamD $ - D.zipWithM f (Par.mkParallelD $ D.toStreamD m1) - (Par.mkParallelD $ D.toStreamD m2) +{-# INLINE zipAsyncWithMK #-} +zipAsyncWithMK :: MonadAsync m + => (a -> b -> m c) -> Stream m a -> Stream m b -> Stream m c +zipAsyncWithMK f m1 m2 = D.toStreamK $ + D.zipWithM f (Par.mkParallelD $ D.fromStreamK m1) + (Par.mkParallelD $ D.fromStreamK m2) -- XXX Should we rename this to zipParWith or zipParallelWith? This can happen -- along with the change of behvaior to end the stream concurrently. @@ -136,10 +115,10 @@ zipAsyncWithM f m1 m2 = D.fromStreamD $ -- as soon as any of the stream end is detected. -- -- @since 0.1.0 -{-# INLINE zipAsyncWith #-} -zipAsyncWith :: (IsStream t, MonadAsync m) - => (a -> b -> c) -> t m a -> t m b -> t m c -zipAsyncWith f = zipAsyncWithM (\a b -> return (f a b)) +{-# INLINE zipAsyncWithK #-} +zipAsyncWithK :: MonadAsync m + => (a -> b -> c) -> Stream m a -> Stream m b -> Stream m c +zipAsyncWithK f = zipAsyncWithMK (\a b -> return (f a b)) ------------------------------------------------------------------------------ -- Serially Zipping Streams @@ -178,49 +157,21 @@ type ZipStream = ZipSerialM -- @since 0.8.0 type ZipSerial = ZipSerialM IO --- | Fix the type of a polymorphic stream as 'ZipSerialM'. --- --- /Since: 0.2.0 ("Streamly")/ --- --- @since 0.8.0 -fromZipSerial :: IsStream t => ZipSerialM m a -> t m a -fromZipSerial = K.adapt - --- | Same as 'fromZipSerial'. --- --- @since 0.1.0 -{-# DEPRECATED zipping "Please use fromZipSerial instead." #-} -zipping :: IsStream t => ZipSerialM m a -> t m a -zipping = fromZipSerial - consMZip :: Monad m => m a -> ZipSerialM m a -> ZipSerialM m a -consMZip m ms = fromStream $ K.consMStream m (toStream ms) - -instance IsStream ZipSerialM where - toStream = getZipSerialM - fromStream = ZipSerialM - - {-# INLINE consM #-} - {-# SPECIALIZE consM :: IO a -> ZipSerialM IO a -> ZipSerialM IO a #-} - consM :: Monad m => m a -> ZipSerialM m a -> ZipSerialM m a - consM = consMZip - - {-# INLINE (|:) #-} - {-# SPECIALIZE (|:) :: IO a -> ZipSerialM IO a -> ZipSerialM IO a #-} - (|:) :: Monad m => m a -> ZipSerialM m a -> ZipSerialM m a - (|:) = consMZip +consMZip m (ZipSerialM r) = ZipSerialM $ K.consM m r LIST_INSTANCES(ZipSerialM) NFDATA1_INSTANCE(ZipSerialM) instance Monad m => Functor (ZipSerialM m) where {-# INLINE fmap #-} - fmap f (ZipSerialM m) = D.fromStreamD $ D.mapM (return . f) $ D.toStreamD m + fmap f (ZipSerialM m) = ZipSerialM $ getSerialT $ fmap f (SerialT m) instance Monad m => Applicative (ZipSerialM m) where - pure = ZipSerialM . K.repeat + pure = ZipSerialM . getSerialT . Serial.repeat + {-# INLINE (<*>) #-} - (<*>) = zipWith id + ZipSerialM m1 <*> ZipSerialM m2 = ZipSerialM $ zipWithK id m1 m2 FOLDABLE_INSTANCE(ZipSerialM) TRAVERSABLE_INSTANCE(ZipSerialM) @@ -257,43 +208,15 @@ newtype ZipAsyncM m a = ZipAsyncM {getZipAsyncM :: Stream m a} -- @since 0.8.0 type ZipAsync = ZipAsyncM IO --- | Fix the type of a polymorphic stream as 'ZipAsyncM'. --- --- /Since: 0.2.0 ("Streamly")/ --- --- @since 0.8.0 -fromZipAsync :: IsStream t => ZipAsyncM m a -> t m a -fromZipAsync = K.adapt - --- | Same as 'fromZipAsync'. --- --- @since 0.1.0 -{-# DEPRECATED zippingAsync "Please use fromZipAsync instead." #-} -zippingAsync :: IsStream t => ZipAsyncM m a -> t m a -zippingAsync = fromZipAsync - consMZipAsync :: Monad m => m a -> ZipAsyncM m a -> ZipAsyncM m a -consMZipAsync m ms = fromStream $ K.consMStream m (toStream ms) - -instance IsStream ZipAsyncM where - toStream = getZipAsyncM - fromStream = ZipAsyncM - - {-# INLINE consM #-} - {-# SPECIALIZE consM :: IO a -> ZipAsyncM IO a -> ZipAsyncM IO a #-} - consM :: Monad m => m a -> ZipAsyncM m a -> ZipAsyncM m a - consM = consMZipAsync - - {-# INLINE (|:) #-} - {-# SPECIALIZE (|:) :: IO a -> ZipAsyncM IO a -> ZipAsyncM IO a #-} - (|:) :: Monad m => m a -> ZipAsyncM m a -> ZipAsyncM m a - (|:) = consMZipAsync +consMZipAsync m (ZipAsyncM r) = ZipAsyncM $ K.consM m r instance Monad m => Functor (ZipAsyncM m) where {-# INLINE fmap #-} - fmap f (ZipAsyncM m) = D.fromStreamD $ D.mapM (return . f) $ D.toStreamD m + fmap f (ZipAsyncM m) = ZipAsyncM $ getSerialT $ fmap f (SerialT m) instance MonadAsync m => Applicative (ZipAsyncM m) where - pure = ZipAsyncM . K.repeat + pure = ZipAsyncM . getSerialT . Serial.repeat + {-# INLINE (<*>) #-} - m1 <*> m2 = zipAsyncWith id m1 m2 + ZipAsyncM m1 <*> ZipAsyncM m2 = ZipAsyncM $ zipAsyncWithK id m1 m2 diff --git a/src/Streamly/Internal/Data/Unfold.hs b/src/Streamly/Internal/Data/Unfold.hs index 3630227955..86eb63ba47 100644 --- a/src/Streamly/Internal/Data/Unfold.hs +++ b/src/Streamly/Internal/Data/Unfold.hs @@ -255,14 +255,15 @@ import Streamly.Internal.Control.Concurrent (MonadAsync) import Streamly.Internal.Data.Fold.Type (Fold(..)) import Streamly.Internal.Data.IOFinalizer (newIOFinalizer, runIOFinalizer, clearingIOFinalizer) +import Streamly.Internal.Data.Stream.IsStream.Type (IsStream) import Streamly.Internal.Data.Stream.StreamD.Type (Stream(..), Step(..)) import Streamly.Internal.Data.SVar.Type (defState) import qualified Control.Monad.Catch as MC import qualified Data.Tuple as Tuple import qualified Streamly.Internal.Data.Fold.Type as FL +import qualified Streamly.Internal.Data.Stream.IsStream.Type as IsStream import qualified Streamly.Internal.Data.Stream.StreamD.Type as D -import qualified Streamly.Internal.Data.Stream.StreamK as K (uncons) import qualified Streamly.Internal.Data.Stream.StreamK.Type as K import Streamly.Internal.Data.Unfold.Enumeration @@ -485,8 +486,8 @@ fromStreamK = Unfold step return -- /Since: 0.8.0/ -- {-# INLINE_NORMAL fromStream #-} -fromStream :: (K.IsStream t, Monad m) => Unfold m (t m a) a -fromStream = lmap K.toStream fromStreamK +fromStream :: (IsStream t, Monad m) => Unfold m (t m a) a +fromStream = lmap IsStream.toStream fromStreamK ------------------------------------------------------------------------------- -- Unfolds diff --git a/src/Streamly/Internal/FileSystem/Dir.hs b/src/Streamly/Internal/FileSystem/Dir.hs index afe934a44c..cf614501c3 100644 --- a/src/Streamly/Internal/FileSystem/Dir.hs +++ b/src/Streamly/Internal/FileSystem/Dir.hs @@ -68,7 +68,7 @@ import Streamly.Internal.Data.Unfold.Type (Unfold(..)) -- (Array(..), writeNUnsafe, defaultChunkSize, shrinkToFit, -- lpackArraysChunksOf) -- import Streamly.Internal.Data.Stream.Serial (SerialT) -import Streamly.Internal.Data.Stream.StreamK.Type (IsStream) +import Streamly.Internal.Data.Stream.IsStream.Type (IsStream) -- import Streamly.String (encodeUtf8, decodeUtf8, foldLines) -- import qualified Streamly.Data.Fold as FL diff --git a/src/Streamly/Internal/FileSystem/FD.hs b/src/Streamly/Internal/FileSystem/FD.hs index 81e027d785..76214ef784 100644 --- a/src/Streamly/Internal/FileSystem/FD.hs +++ b/src/Streamly/Internal/FileSystem/FD.hs @@ -135,12 +135,11 @@ import Streamly.Internal.Data.Array.Foreign.Type import Streamly.Internal.Data.Array.Foreign.Mut.Type (mutableArray) import Streamly.Internal.System.IO (defaultChunkSize) import Streamly.Internal.Data.Stream.Serial (SerialT) -import Streamly.Internal.Data.Stream.StreamK.Type (IsStream, mkStream) - +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream, mkStream, fromStreamD) #if !defined(mingw32_HOST_OS) +import Streamly.Internal.Data.Stream.IsStream.Type (toStreamD) import Streamly.Internal.System.IOVec (groupIOVecsOf) -import Streamly.Internal.Data.Stream.StreamD (toStreamD) -import Streamly.Internal.Data.Stream.StreamD.Type (fromStreamD) import qualified Streamly.Internal.FileSystem.FDIO as RawIO hiding (write) import qualified Streamly.Internal.System.IOVec.Type as RawIO #endif @@ -290,7 +289,7 @@ _readArraysOfUpto size h = go {-# INLINE_NORMAL readArraysOfUpto #-} readArraysOfUpto :: (IsStream t, MonadIO m) => Int -> Handle -> t m (Array Word8) -readArraysOfUpto size h = D.fromStreamD (D.Stream step ()) +readArraysOfUpto size h = fromStreamD (D.Stream step ()) where {-# INLINE_LATE step #-} step _ _ = do diff --git a/src/Streamly/Internal/FileSystem/File.hs b/src/Streamly/Internal/FileSystem/File.hs index f0402667da..2d78a48233 100644 --- a/src/Streamly/Internal/FileSystem/File.hs +++ b/src/Streamly/Internal/FileSystem/File.hs @@ -99,17 +99,17 @@ import Streamly.Internal.Data.Fold.Type (Fold(..)) import Streamly.Internal.Data.Unfold.Type (Unfold(..)) import Streamly.Internal.Data.Array.Foreign.Type (Array(..), writeNUnsafe) import Streamly.Internal.Data.Stream.Serial (SerialT) -import Streamly.Internal.Data.Stream.StreamK.Type (IsStream) +import Streamly.Internal.Data.Stream.IsStream.Type (IsStream) -- import Streamly.Data.Fold (Fold) -- import Streamly.String (encodeUtf8, decodeUtf8, foldLines) import Streamly.Internal.System.IO (defaultChunkSize) +import qualified Streamly.Internal.Data.Array.Foreign as A import qualified Streamly.Internal.Data.Fold.Type as FL import qualified Streamly.Internal.Data.Unfold as UF import qualified Streamly.Internal.FileSystem.Handle as FH import qualified Streamly.Internal.Data.Array.Stream.Foreign as AS import qualified Streamly.Internal.Data.Stream.IsStream as S -import qualified Streamly.Data.Array.Foreign as A ------------------------------------------------------------------------------- -- References diff --git a/src/Streamly/Internal/FileSystem/Handle.hs b/src/Streamly/Internal/FileSystem/Handle.hs index 6e5e82af4b..168735db50 100644 --- a/src/Streamly/Internal/FileSystem/Handle.hs +++ b/src/Streamly/Internal/FileSystem/Handle.hs @@ -110,25 +110,26 @@ import System.IO (Handle, SeekMode(..), hGetBufSome, hPutBuf, hSeek) import Prelude hiding (read) import Streamly.Internal.BaseCompat -import Streamly.Data.Fold (Fold) +import Streamly.Internal.Data.Fold (Fold) import Streamly.Internal.Data.Fold.Type (Fold2(..)) import Streamly.Internal.Data.Unfold.Type (Unfold(..)) import Streamly.Internal.Data.Array.Foreign.Type (Array(..), writeNUnsafe, unsafeFreezeWithShrink) import Streamly.Internal.Data.Array.Foreign.Mut.Type (mutableArray) import Streamly.Internal.Data.Stream.Serial (SerialT) -import Streamly.Internal.Data.Stream.StreamK.Type (IsStream, mkStream) +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream, mkStream, fromStreamD) import Streamly.Internal.Data.Array.Stream.Foreign (lpackArraysChunksOf) -- import Streamly.String (encodeUtf8, decodeUtf8, foldLines) import Streamly.Internal.System.IO (defaultChunkSize) -import qualified Streamly.Data.Fold as FL -import qualified Streamly.Internal.Data.Fold.Type as FL -import qualified Streamly.Internal.Data.Unfold as UF +import qualified Streamly.Internal.Data.Array.Foreign as A import qualified Streamly.Internal.Data.Array.Stream.Foreign as AS +import qualified Streamly.Internal.Data.Fold as FL +import qualified Streamly.Internal.Data.Fold.Type as FL import qualified Streamly.Internal.Data.Stream.IsStream as S -import qualified Streamly.Data.Array.Foreign as A import qualified Streamly.Internal.Data.Stream.StreamD.Type as D +import qualified Streamly.Internal.Data.Unfold as UF -- $setup -- >>> import qualified Streamly.Data.Array.Foreign as Array @@ -219,7 +220,7 @@ _toChunksWithBufferOf size h = go {-# INLINE_NORMAL toChunksWithBufferOf #-} toChunksWithBufferOf :: (IsStream t, MonadIO m) => Int -> Handle -> t m (Array Word8) -toChunksWithBufferOf size h = D.fromStreamD (D.Stream step ()) +toChunksWithBufferOf size h = fromStreamD (D.Stream step ()) where {-# INLINE_LATE step #-} step _ _ = do diff --git a/src/Streamly/Internal/Network/Inet/TCP.hs b/src/Streamly/Internal/Network/Inet/TCP.hs index d85a9bdba9..714e6d3eb3 100644 --- a/src/Streamly/Internal/Network/Inet/TCP.hs +++ b/src/Streamly/Internal/Network/Inet/TCP.hs @@ -106,8 +106,8 @@ import Prelude hiding (read) import Streamly.Internal.Control.Concurrent (MonadAsync, fork) import Streamly.Internal.Data.Array.Foreign.Type (Array(..), writeNUnsafe) import Streamly.Internal.Data.Fold.Type (Fold(..)) +import Streamly.Internal.Data.Stream.IsStream.Type (IsStream) import Streamly.Internal.Data.Stream.Serial (SerialT) -import Streamly.Internal.Data.Stream.StreamK.Type (IsStream) import Streamly.Internal.Data.Tuple.Strict (Tuple'(..)) import Streamly.Internal.Data.Unfold.Type (Unfold(..)) import Streamly.Internal.Network.Socket (SockSpec(..), accept, connections) @@ -121,7 +121,6 @@ import qualified Streamly.Internal.Data.Array.Foreign as A import qualified Streamly.Internal.Data.Array.Stream.Foreign as AS import qualified Streamly.Internal.Data.Fold.Type as FL import qualified Streamly.Internal.Data.Stream.IsStream as S -import qualified Streamly.Network.Socket as SK import qualified Streamly.Internal.Network.Socket as ISK ------------------------------------------------------------------------------- @@ -356,7 +355,8 @@ writeChunks addr port = Fold step initial extract where initial = do skt <- liftIO (connect addr port) - fld <- FL.initialize (SK.writeChunks skt) `MC.onException` liftIO (Net.close skt) + fld <- FL.initialize (ISK.writeChunks skt) + `MC.onException` liftIO (Net.close skt) return $ FL.Partial (Tuple' fld skt) step (Tuple' fld skt) x = do r <- FL.runStep fld x `MC.onException` liftIO (Net.close skt) diff --git a/src/Streamly/Internal/Network/Socket.hs b/src/Streamly/Internal/Network/Socket.hs index 1623bb14b8..d9a3b699e8 100644 --- a/src/Streamly/Internal/Network/Socket.hs +++ b/src/Streamly/Internal/Network/Socket.hs @@ -86,24 +86,25 @@ import Prelude hiding (read) import qualified Network.Socket as Net import Streamly.Internal.BaseCompat -import Streamly.Internal.Data.Unfold.Type (Unfold(..)) -import Streamly.Internal.Data.Array.Stream.Foreign (lpackArraysChunksOf) -import Streamly.Internal.Data.Array.Foreign.Type (Array(..)) +import Streamly.Internal.Control.Concurrent (MonadAsync) import Streamly.Internal.Data.Array.Foreign.Mut.Type (mutableArray) -import Streamly.Internal.Data.Stream.IsStream (MonadAsync) +import Streamly.Internal.Data.Array.Foreign.Type (Array(..)) +import Streamly.Internal.Data.Array.Stream.Foreign (lpackArraysChunksOf) +import Streamly.Internal.Data.Fold (Fold) +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream, mkStream, fromStreamD) import Streamly.Internal.Data.Stream.Serial (SerialT) -import Streamly.Internal.Data.Stream.StreamK.Type (IsStream, mkStream) -import Streamly.Data.Fold (Fold) +import Streamly.Internal.Data.Unfold.Type (Unfold(..)) -- import Streamly.String (encodeUtf8, decodeUtf8, foldLines) import Streamly.Internal.System.IO (defaultChunkSize) -import qualified Streamly.Data.Fold as FL -import qualified Streamly.Internal.Data.Unfold as UF -import qualified Streamly.Data.Array.Foreign as A -import qualified Streamly.Internal.Data.Array.Stream.Foreign as AS +import qualified Streamly.Internal.Data.Array.Foreign as A import qualified Streamly.Internal.Data.Array.Foreign.Type as A +import qualified Streamly.Internal.Data.Array.Stream.Foreign as AS +import qualified Streamly.Internal.Data.Fold as FL import qualified Streamly.Internal.Data.Stream.IsStream as S import qualified Streamly.Internal.Data.Stream.StreamD.Type as D +import qualified Streamly.Internal.Data.Unfold as UF -- | @'forSocketM' action socket@ runs the monadic computation @action@ passing -- the socket handle to it. The handle will be closed on exit from @@ -343,7 +344,7 @@ _readChunksUptoWith f size h = go toChunksWithBufferOf :: (IsStream t, MonadIO m) => Int -> Socket -> t m (Array Word8) -- toChunksWithBufferOf = _readChunksUptoWith readChunk -toChunksWithBufferOf size h = D.fromStreamD (D.Stream step ()) +toChunksWithBufferOf size h = fromStreamD (D.Stream step ()) where {-# INLINE_LATE step #-} step _ _ = do diff --git a/src/Streamly/Internal/Unicode/Char.hs b/src/Streamly/Internal/Unicode/Char.hs index 26377640ee..18b3e1cb94 100644 --- a/src/Streamly/Internal/Unicode/Char.hs +++ b/src/Streamly/Internal/Unicode/Char.hs @@ -32,14 +32,11 @@ import Data.Char (isAsciiUpper, isAsciiLower, chr, ord) import Data.Typeable (Typeable) import Unicode.Char (DecomposeMode(..)) -import Streamly.Internal.Data.Stream.IsStream (IsStream) +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream, fromStreamD, toStreamD) import Streamly.Internal.Data.Stream.StreamD (Stream(..), Step (..)) import qualified Unicode.Char as Char -import qualified Streamly.Internal.Data.Stream.StreamD as StreamD - - --- import Streamly.Prelude (IsStream) ------------------------------------------------------------------------------- -- Unicode aware operations on strings @@ -319,4 +316,4 @@ normalizeD NFC = partialComposeD . decomposeD False Canonical normalizeD NFKC = partialComposeD . decomposeD False Kompat normalize :: (IsStream t, Monad m) => NormalizationMode -> t m Char -> t m Char -normalize mode = StreamD.fromStreamD . normalizeD mode . StreamD.toStreamD +normalize mode = fromStreamD . normalizeD mode . toStreamD diff --git a/src/Streamly/Internal/Unicode/Stream.hs b/src/Streamly/Internal/Unicode/Stream.hs index 3ced99df3a..0590be9257 100644 --- a/src/Streamly/Internal/Unicode/Stream.hs +++ b/src/Streamly/Internal/Unicode/Stream.hs @@ -90,14 +90,15 @@ import GHC.ForeignPtr (ForeignPtr (..)) import GHC.IO.Encoding.Failure (isSurrogate) import GHC.Ptr (Ptr (..), plusPtr) import System.IO.Unsafe (unsafePerformIO) -import Streamly.Data.Fold (Fold) -import Streamly.Data.Array.Foreign (Array) -import Streamly.Internal.Data.Unfold (Unfold) -import Streamly.Internal.Data.SVar (adaptState) -import Streamly.Internal.Data.Stream.IsStream (IsStream) +import Streamly.Internal.Data.Array.Foreign (Array) +import Streamly.Internal.Data.Fold (Fold) +import Streamly.Internal.Data.Stream.IsStream.Type + (IsStream, fromStreamD, toStreamD, adapt) import Streamly.Internal.Data.Stream.Serial (SerialT) import Streamly.Internal.Data.Stream.StreamD (Stream(..), Step (..)) +import Streamly.Internal.Data.SVar (adaptState) import Streamly.Internal.Data.Tuple.Strict (Tuple'(..)) +import Streamly.Internal.Data.Unfold (Unfold) import Streamly.Internal.System.IO (unsafeInlineIO) import qualified Streamly.Internal.Data.Unfold as Unfold @@ -396,7 +397,7 @@ decodeUtf8EitherD = resumeDecodeUtf8EitherD 0 0 {-# INLINE decodeUtf8Either #-} decodeUtf8Either :: (Monad m, IsStream t) => t m Word8 -> t m (Either DecodeError Char) -decodeUtf8Either = D.fromStreamD . decodeUtf8EitherD . D.toStreamD +decodeUtf8Either = fromStreamD . decodeUtf8EitherD . toStreamD -- | -- @@ -409,7 +410,7 @@ resumeDecodeUtf8Either -> t m Word8 -> t m (Either DecodeError Char) resumeDecodeUtf8Either st cp = - D.fromStreamD . resumeDecodeUtf8EitherD st cp . D.toStreamD + fromStreamD . resumeDecodeUtf8EitherD st cp . toStreamD ------------------------------------------------------------------------------- -- One shot decoding @@ -533,7 +534,7 @@ decodeUtf8D = decodeUtf8WithD TransliterateCodingFailure -- /Since: 0.8.0 (Lenient Behaviour)/ {-# INLINE decodeUtf8 #-} decodeUtf8 :: (Monad m, IsStream t) => t m Word8 -> t m Char -decodeUtf8 = D.fromStreamD . decodeUtf8D . D.toStreamD +decodeUtf8 = fromStreamD . decodeUtf8D . toStreamD {-# INLINE decodeUtf8D' #-} decodeUtf8D' :: Monad m => Stream m Word8 -> Stream m Char @@ -545,7 +546,7 @@ decodeUtf8D' = decodeUtf8WithD ErrorOnCodingFailure -- @since 0.8.0 {-# INLINE decodeUtf8' #-} decodeUtf8' :: (Monad m, IsStream t) => t m Word8 -> t m Char -decodeUtf8' = D.fromStreamD . decodeUtf8D' . D.toStreamD +decodeUtf8' = fromStreamD . decodeUtf8D' . toStreamD {-# INLINE decodeUtf8D_ #-} decodeUtf8D_ :: Monad m => Stream m Word8 -> Stream m Char @@ -557,7 +558,7 @@ decodeUtf8D_ = decodeUtf8WithD DropOnCodingFailure -- @since 0.8.0 {-# INLINE decodeUtf8_ #-} decodeUtf8_ :: (Monad m, IsStream t) => t m Word8 -> t m Char -decodeUtf8_ = D.fromStreamD . decodeUtf8D_ . D.toStreamD +decodeUtf8_ = fromStreamD . decodeUtf8D_ . toStreamD -- | Same as 'decodeUtf8' -- @@ -715,7 +716,7 @@ decodeUtf8ArraysD = decodeUtf8ArraysWithD TransliterateCodingFailure decodeUtf8Arrays :: (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char decodeUtf8Arrays = - D.fromStreamD . decodeUtf8ArraysD . D.toStreamD + fromStreamD . decodeUtf8ArraysD . toStreamD {-# INLINE decodeUtf8ArraysD' #-} decodeUtf8ArraysD' :: @@ -729,7 +730,7 @@ decodeUtf8ArraysD' = decodeUtf8ArraysWithD ErrorOnCodingFailure -- /Pre-release/ {-# INLINE decodeUtf8Arrays' #-} decodeUtf8Arrays' :: (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char -decodeUtf8Arrays' = D.fromStreamD . decodeUtf8ArraysD' . D.toStreamD +decodeUtf8Arrays' = fromStreamD . decodeUtf8ArraysD' . toStreamD {-# INLINE decodeUtf8ArraysD_ #-} decodeUtf8ArraysD_ :: @@ -745,7 +746,7 @@ decodeUtf8ArraysD_ = decodeUtf8ArraysWithD DropOnCodingFailure decodeUtf8Arrays_ :: (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char decodeUtf8Arrays_ = - D.fromStreamD . decodeUtf8ArraysD_ . D.toStreamD + fromStreamD . decodeUtf8ArraysD_ . toStreamD ------------------------------------------------------------------------------- -- Encoding Unicode (UTF-8) Characters @@ -867,7 +868,7 @@ encodeUtf8D' = encodeUtf8DGeneric ErrorInvalid -- @since 0.8.0 {-# INLINE encodeUtf8' #-} encodeUtf8' :: (Monad m, IsStream t) => t m Char -> t m Word8 -encodeUtf8' = D.fromStreamD . encodeUtf8D' . D.toStreamD +encodeUtf8' = fromStreamD . encodeUtf8D' . toStreamD -- | See section "3.9 Unicode Encoding Forms" in -- https://www.unicode.org/versions/Unicode13.0.0/UnicodeStandard-13.0.pdf @@ -885,7 +886,7 @@ encodeUtf8D = encodeUtf8DGeneric ReplaceInvalid -- /Since: 0.8.0 (Lenient Behaviour)/ {-# INLINE encodeUtf8 #-} encodeUtf8 :: (Monad m, IsStream t) => t m Char -> t m Word8 -encodeUtf8 = D.fromStreamD . encodeUtf8D . D.toStreamD +encodeUtf8 = fromStreamD . encodeUtf8D . toStreamD {-# INLINE_NORMAL encodeUtf8D_ #-} encodeUtf8D_ :: Monad m => Stream m Char -> Stream m Word8 @@ -897,7 +898,7 @@ encodeUtf8D_ = encodeUtf8DGeneric DropInvalid -- @since 0.8.0 {-# INLINE encodeUtf8_ #-} encodeUtf8_ :: (Monad m, IsStream t) => t m Char -> t m Word8 -encodeUtf8_ = D.fromStreamD . encodeUtf8D_ . D.toStreamD +encodeUtf8_ = fromStreamD . encodeUtf8D_ . toStreamD -- | Same as 'encodeUtf8' -- @@ -932,7 +933,7 @@ encodeObjects :: (MonadIO m, IsStream t) => -> Unfold m a Char -> t m a -> t m (Array Word8) -encodeObjects encode u = Serial.mapM (encodeObject encode u) +encodeObjects encode u = adapt . Serial.mapM (encodeObject encode u) . adapt -- | Encode a stream of 'String' using the supplied encoding scheme. Each -- string is encoded as an @Array Word8@. diff --git a/src/Streamly/Prelude.hs b/src/Streamly/Prelude.hs index f1e46746d2..efb1b1f919 100644 --- a/src/Streamly/Prelude.hs +++ b/src/Streamly/Prelude.hs @@ -936,6 +936,7 @@ module Streamly.Prelude ) where +import Streamly.Internal.Control.Concurrent (MonadAsync) import Prelude hiding (filter, drop, dropWhile, take, takeWhile, zipWith, foldr, foldl, map, mapM, mapM_, sequence, all, any, sum, product, elem, diff --git a/streamly.cabal b/streamly.cabal index 0d79882501..ae68afb0ef 100644 --- a/streamly.cabal +++ b/streamly.cabal @@ -77,6 +77,7 @@ extra-source-files: docs/*.hs docs/streamly-docs.cabal src/Streamly/Internal/Data/Stream/Instances.hs + src/Streamly/Internal/Data/Stream/PreludeCommon.hs src/Streamly/Internal/Data/Time/Clock/config-clock.h src/Streamly/Internal/Data/Array/PrimInclude.hs src/Streamly/Internal/Data/Array/Prim/TypesInclude.hs @@ -263,6 +264,7 @@ common default-extensions RecordWildCards ScopedTypeVariables TupleSections + TypeApplications TypeFamilies ViewPatterns @@ -301,13 +303,11 @@ common lib-options library import: lib-options - if impl(ghc >= 8.1) - default-extensions: TypeApplications if impl(ghc >= 8.6) default-extensions: QuantifiedConstraints js-sources: jsbits/clock.js - include-dirs: src + include-dirs: src, src/Streamly/Internal/Data/Stream if os(windows) c-sources: src/Streamly/Internal/Data/Time/Clock/Windows.c exposed-modules: Streamly.Internal.FileSystem.Event.Windows @@ -412,7 +412,6 @@ library , Streamly.Internal.Data.SmallArray.Type -- streamly-base-streams - , Streamly.Internal.Data.Stream.StreamK -- StreamD depends on streamly-core-array , Streamly.Internal.Data.Stream.StreamD.Generate , Streamly.Internal.Data.Stream.StreamD.Eliminate @@ -434,6 +433,7 @@ library , Streamly.Internal.Data.Fold.Tee , Streamly.Internal.Data.Fold , Streamly.Internal.Data.Fold.SVar + , Streamly.Internal.Data.Fold.Async , Streamly.Internal.Data.Sink , Streamly.Internal.Data.Parser , Streamly.Internal.Data.Pipe @@ -445,9 +445,11 @@ library , Streamly.Internal.Data.Stream.Parallel , Streamly.Internal.Data.Stream.Ahead , Streamly.Internal.Data.Stream.Zip + , Streamly.Internal.Data.List + + , Streamly.Internal.Data.Stream.IsStream.Type , Streamly.Internal.Data.Stream.IsStream.Combinators , Streamly.Internal.Data.Stream.IsStream.Common - , Streamly.Internal.Data.Stream.IsStream.Types , Streamly.Internal.Data.Stream.IsStream.Enumeration , Streamly.Internal.Data.Stream.IsStream.Generate , Streamly.Internal.Data.Stream.IsStream.Eliminate @@ -458,7 +460,6 @@ library , Streamly.Internal.Data.Stream.IsStream.Lift , Streamly.Internal.Data.Stream.IsStream.Top , Streamly.Internal.Data.Stream.IsStream - , Streamly.Internal.Data.List -- streamly-arrays -- May depend on streamly-core @@ -501,6 +502,9 @@ library , Streamly.Internal.Network.Socket , Streamly.Internal.Network.Inet.TCP + -- Only used for benchmarks + , Streamly.Internal.Data.Stream.StreamK + build-depends: -- Core libraries shipped with ghc, the min and max -- constraints of these libraries should match with diff --git a/test/Streamly/Test/Prelude/WSerial.hs b/test/Streamly/Test/Prelude/WSerial.hs index bfd935b50e..5cf5d00903 100644 --- a/test/Streamly/Test/Prelude/WSerial.hs +++ b/test/Streamly/Test/Prelude/WSerial.hs @@ -60,7 +60,8 @@ interleaveCheck t f = wSerialMinLengthProp :: Property wSerialMinLengthProp = forAll (chooseInt (0, 10)) - $ \len -> S.length (combined len) `shouldReturn` 2 * len + 1 + $ \len -> + S.length (fromWSerial $ combined len) `shouldReturn` 2 * len + 1 where From 9d7bd5293aff7a4bf06d32bb0fe4a398d5330b48 Mon Sep 17 00:00:00 2001 From: Harendra Kumar Date: Mon, 13 Sep 2021 11:06:16 +0530 Subject: [PATCH 3/6] Make the test more reliable and add diag info --- test/Streamly/Test/Prelude/Fold.hs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/test/Streamly/Test/Prelude/Fold.hs b/test/Streamly/Test/Prelude/Fold.hs index f0cc130a9c..6761b024a2 100644 --- a/test/Streamly/Test/Prelude/Fold.hs +++ b/test/Streamly/Test/Prelude/Fold.hs @@ -13,12 +13,13 @@ module Streamly.Test.Prelude.Fold where import Control.Concurrent (threadDelay) #endif import Control.Exception (ErrorCall(..), catch) -import Data.IORef ( newIORef, readIORef, writeIORef, IORef ) +import Data.IORef (newIORef, readIORef, writeIORef, IORef) #ifdef COVERAGE_BUILD import Test.Hspec.QuickCheck (modifyMaxSuccess) #endif import Test.Hspec as H #ifdef DEVBUILD +import Data.IORef (modifyIORef) import System.Mem (performMajorGC) #endif @@ -63,14 +64,20 @@ checkCleanupFold :: IsStream t -> (SerialT IO Int -> IO (Maybe Int)) -> IO () checkCleanupFold t op = do - r <- newIORef (-1 :: Int) + r <- newIORef ([] :: [Int]) _ <- op $ t $ delay r 0 S.|: delay r 2 S.|: delay r 3 S.|: S.nil performMajorGC - threadDelay 700000 + -- In case the actions are not killed let them fire before we check so that + -- the test fails. + threadDelay (4 * delayUnit) res <- readIORef r - res `shouldBe` 0 + res `shouldBe` [0] + where - delay ref i = threadDelay (i*200000) >> writeIORef ref i >> return i + + delayUnit = 400000 + delay ref i = + threadDelay (i * delayUnit) >> modifyIORef ref (i :) >> return i testFoldOpsCleanup :: String -> (SerialT IO Int -> IO a) -> Spec testFoldOpsCleanup name f = do From ea50a6cb50d9d2202d1640e02ab22a90bc1d71e9 Mon Sep 17 00:00:00 2001 From: Harendra Kumar Date: Tue, 14 Sep 2021 22:31:55 +0530 Subject: [PATCH 4/6] Add use-large-mem flag To include tests that require large amounts of RAM --- test/lib/Streamly/Test/Prelude/Common.hs | 2 ++ test/streamly-tests.cabal | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/test/lib/Streamly/Test/Prelude/Common.hs b/test/lib/Streamly/Test/Prelude/Common.hs index 5098b57dbd..6166b0f7bb 100644 --- a/test/lib/Streamly/Test/Prelude/Common.hs +++ b/test/lib/Streamly/Test/Prelude/Common.hs @@ -1754,7 +1754,9 @@ makeCommonOps t = , ("maxBuffer 0", t . maxBuffer 0) , ("maxThreads 0", t . maxThreads 0) , ("maxThreads 1", t . maxThreads 1) +#ifdef USE_LARGE_MEMORY , ("maxThreads -1", t . maxThreads (-1)) +#endif #endif ] diff --git a/test/streamly-tests.cabal b/test/streamly-tests.cabal index e911933298..a8312febb7 100644 --- a/test/streamly-tests.cabal +++ b/test/streamly-tests.cabal @@ -15,6 +15,11 @@ flag limit-build-mem manual: True default: False +flag use-large-mem + description: Include tests that require large amount of memory + manual: True + default: False + flag dev description: Development build manual: True @@ -70,6 +75,9 @@ common compile-options if flag(limit-build-mem) ghc-options: +RTS -M512M -RTS + if flag(use-large-mem) + cpp-options: -DUSE_LARGE_MEMORY + common default-extensions default-extensions: BangPatterns From b82985820efb48adf63583fa00336fbfb0261b66 Mon Sep 17 00:00:00 2001 From: Harendra Kumar Date: Mon, 13 Sep 2021 21:53:08 +0530 Subject: [PATCH 5/6] Disable rate tests taking too much memory --- bin/test.sh | 1 + test/Streamly/Test/Prelude/Rate.hs | 32 ++++++++++++++++-------------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/bin/test.sh b/bin/test.sh index 5943cde9ea..837ece535e 100755 --- a/bin/test.sh +++ b/bin/test.sh @@ -107,6 +107,7 @@ test_exe_rts_opts () { Data.Array.Foreign) echo -n "-M128M" ;; Data.Array.Prim) echo -n "-M128M" ;; Data.Array.Prim.Pinned) echo -n "-M128M" ;; + Prelude.Rate) echo -n "-M512M" ;; # For -O0 case writeChunks test fails, maybe we should have a separate flag # for O0 case? FileSystem.Handle) echo -n "-K16M" ;; diff --git a/test/Streamly/Test/Prelude/Rate.hs b/test/Streamly/Test/Prelude/Rate.hs index 028229eb2f..e6d1ffc551 100644 --- a/test/Streamly/Test/Prelude/Rate.hs +++ b/test/Streamly/Test/Prelude/Rate.hs @@ -177,9 +177,11 @@ main = hspec $ do forM_ rates (\r -> measureRate "asyncly" fromAsync r 0 0 range) -- XXX try staggering the dispatches to achieve higher rates - let rates = [1, 10, 100, 1000 -#ifndef __GHCJS__ - , 10000, 25000 + -- Producer delay causes a lot of threads to be created, consuming large + -- amounts of memory at higher rates. + let rates = [1, 10, 100 +#if !defined(__GHCJS__) && defined USE_LARGE_MEMORY + 1000, 10000, 25000 #endif ] in describe "asyncly no consumer delay and 1 sec producer delay" $ @@ -187,12 +189,12 @@ main = hspec $ do -- At lower rates (1/10) this is likely to vary quite a bit depending on -- the spread of random producer latencies generated. - let rates = [1, 10, 100, 1000 -#ifndef __GHCJS__ - , 10000, 25000 + let rates = [1, 10, 100 +#if !defined(__GHCJS__) && defined USE_LARGE_MEMORY + , 1000, 10000, 25000 #endif ] - in describe "asyncly no consumer delay and variable producer delay" $ + in describe "asyncly, no consumer delay and variable producer delay" $ forM_ rates $ \r -> measureRateVariable "asyncly" fromAsync r 0 (0.1, 3) range @@ -201,15 +203,15 @@ main = hspec $ do , 100000, 1000000 #endif ] - in describe "fromWAsync no consumer delay no producer delay" $ + in describe "fromWAsync, no consumer delay no producer delay" $ forM_ rates (\r -> measureRate "fromWAsync" fromWAsync r 0 0 range) let rates = [1, 10, 100, 1000 -#ifndef __GHCJS__ +#if !defined(__GHCJS__) && defined USE_LARGE_MEMORY , 10000, 25000 #endif ] - in describe "fromWAsync no consumer delay and 1 sec producer delay" $ + in describe "fromWAsync, no consumer delay and 1 sec producer delay" $ forM_ rates (\r -> measureRate "fromWAsync" fromWAsync r 0 1 range) let rates = [1, 10, 100, 1000, 10000 @@ -217,20 +219,20 @@ main = hspec $ do , 100000, 1000000 #endif ] - in describe "aheadly no consumer delay no producer delay" $ + in describe "aheadly, no consumer delay no producer delay" $ forM_ rates (\r -> measureRate "aheadly" fromAhead r 0 0 range) -- XXX after the change to stop workers when the heap is clearing -- thi does not work well at a 25000 ops per second, need to fix. let rates = [1, 10, 100, 1000 -#ifndef __GHCJS__ +#if !defined(__GHCJS__) && defined USE_LARGE_MEMORY , 10000, 12500 #endif ] - in describe "aheadly no consumer delay and 1 sec producer delay" $ + in describe "aheadly, no consumer delay and 1 sec producer delay" $ forM_ rates (\r -> measureRate "aheadly" fromAhead r 0 1 range) - describe "asyncly with 1 sec producer delay and some consumer delay" $ do + describe "asyncly, some consumer delay and 1 sec producer delay" $ do -- ideally it should take 10 x 1 + 1 seconds forM_ [1] (\r -> measureRate "asyncly" fromAsync r 1 1 (11, 16)) -- ideally it should take 10 x 2 + 1 seconds @@ -238,7 +240,7 @@ main = hspec $ do -- ideally it should take 10 x 3 + 1 seconds forM_ [1] (\r -> measureRate "asyncly" fromAsync r 3 1 (31, 33)) - describe "aheadly with 1 sec producer delay and some consumer delay" $ do + describe "aheadly, some consumer delay and 1 sec producer delay" $ do forM_ [1] (\r -> measureRate "aheadly" fromAhead r 1 1 (11, 16)) forM_ [1] (\r -> measureRate "aheadly" fromAhead r 2 1 (21, 23)) forM_ [1] (\r -> measureRate "aheadly" fromAhead r 3 1 (31, 33)) From f6b88dc488786d75d2652f972dea2aa21bb3308b Mon Sep 17 00:00:00 2001 From: Harendra Kumar Date: Tue, 14 Sep 2021 23:19:54 +0530 Subject: [PATCH 6/6] Add a flag for flaky tests --- test/Streamly/Test/Prelude/Fold.hs | 2 ++ test/lib/Streamly/Test/Prelude/Common.hs | 4 ++-- test/streamly-tests.cabal | 8 ++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/test/Streamly/Test/Prelude/Fold.hs b/test/Streamly/Test/Prelude/Fold.hs index 6761b024a2..3278ad6036 100644 --- a/test/Streamly/Test/Prelude/Fold.hs +++ b/test/Streamly/Test/Prelude/Fold.hs @@ -85,8 +85,10 @@ testFoldOpsCleanup name f = do it (name <> " asyncly") $ checkCleanupFold S.fromAsync (testOp f) it (name <> " wAsyncly") $ checkCleanupFold S.fromWAsync (testOp f) it (name <> " aheadly") $ checkCleanupFold S.fromAhead (testOp f) +#ifdef INCLUDE_FLAKY_TESTS it (name <> " parallely") $ checkCleanupFold S.fromParallel (testOp f) #endif +#endif checkFoldMStrictness :: (IORef Int -> SerialT IO Int -> IO ()) -> IO () checkFoldMStrictness f = do diff --git a/test/lib/Streamly/Test/Prelude/Common.hs b/test/lib/Streamly/Test/Prelude/Common.hs index 6166b0f7bb..22594b1664 100644 --- a/test/lib/Streamly/Test/Prelude/Common.hs +++ b/test/lib/Streamly/Test/Prelude/Common.hs @@ -1628,13 +1628,13 @@ exceptionOps desc t = do prop (desc <> " before") $ beforeProp t prop (desc <> " after") $ afterProp t prop (desc <> " bracket end of stream") $ bracketProp t -#ifdef DEVBUILD +#ifdef INCLUDE_FLAKY_TESTS prop (desc <> " bracket partial stream") $ bracketPartialStreamProp t #endif prop (desc <> " bracket exception in stream") $ bracketExceptionProp t prop (desc <> " onException") $ onExceptionProp t prop (desc <> " finally end of stream") $ finallyProp t -#ifdef DEVBUILD +#ifdef INCLUDE_FLAKY_TESTS prop (desc <> " finally partial stream") $ finallyPartialStreamProp t #endif prop (desc <> " finally exception in stream") $ finallyExceptionProp t diff --git a/test/streamly-tests.cabal b/test/streamly-tests.cabal index a8312febb7..efa27b91a3 100644 --- a/test/streamly-tests.cabal +++ b/test/streamly-tests.cabal @@ -20,6 +20,11 @@ flag use-large-mem manual: True default: False +flag include-flaky-tests + description: Include tests that are unpredictable + manual: True + default: False + flag dev description: Development build manual: True @@ -78,6 +83,9 @@ common compile-options if flag(use-large-mem) cpp-options: -DUSE_LARGE_MEMORY + if flag(include-flaky-tests) + cpp-options: -DINCLUDE_FLAKY_TESTS + common default-extensions default-extensions: BangPatterns