fraport/fradrive-old
Archived
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(inject-files): additionally throttle by file count

Browse Source
This commit is contained in:
Gregor Kleen 2020-12-04 18:08:10 +01:00
parent c22004e1b2
commit 3cf033560e
3 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config/settings.yml
View File

@ -242,6 +242,10 @@ token-buckets:
depth: 20971520 # 20MiB depth: 20971520 # 20MiB
inv-rate: 9.5e-7 # 1MiB/s inv-rate: 9.5e-7 # 1MiB/s
initial-value: 0 initial-value: 0
inject-files-count:
depth: 100
inv-rate: 1
initial-value: 0
prune-files: prune-files:
depth: 1572864000 # 1500MiB depth: 1572864000 # 1500MiB
inv-rate: 1.9e-6 # 2MiB/s inv-rate: 1.9e-6 # 2MiB/s

1
src/Jobs/Handler/Files.hs
View File

@ -370,6 +370,7 @@ dispatchJobInjectFiles = JobHandlerException . maybeT (return ()) $ do
.| C.mapMaybe extractReference .| C.mapMaybe extractReference
.| maybe (C.map id) (takeWhileTime . (/ 2)) interval .| maybe (C.map id) (takeWhileTime . (/ 2)) interval
.| transPipe (lift . runDB . setSerializable) (persistentTokenBucketTakeC' TokenBucketInjectFiles $ views _1 Minio.oiSize) .| transPipe (lift . runDB . setSerializable) (persistentTokenBucketTakeC' TokenBucketInjectFiles $ views _1 Minio.oiSize)
.| transPipe (lift . runDB . setSerializable) (persistentTokenBucketTakeC' TokenBucketInjectFilesCount $ const 1)
.| C.mapM (lift . injectOrDelete) .| C.mapM (lift . injectOrDelete)
.| C.mapM (\res@(Sum inj, Sum sz) -> res <$ observeInjectedFiles inj sz) .| C.mapM (\res@(Sum inj, Sum sz) -> res <$ observeInjectedFiles inj sz)
.| C.fold .| C.fold

2
src/Model/Types/Misc.hs
View File

@ -265,7 +265,7 @@ instance Csv.FromField Sex where
parseField = maybe (fail "Could not parse Field of type Sex") return . fromPathPiece <=< Csv.parseField parseField = maybe (fail "Could not parse Field of type Sex") return . fromPathPiece <=< Csv.parseField
data TokenBucketIdent = TokenBucketInjectFiles data TokenBucketIdent = TokenBucketInjectFiles | TokenBucketInjectFilesCount
| TokenBucketPruneFiles | TokenBucketPruneFiles
| TokenBucketRechunkFiles | TokenBucketRechunkFiles
deriving (Eq, Ord, Read, Show, Enum, Bounded, Generic, Typeable) deriving (Eq, Ord, Read, Show, Enum, Bounded, Generic, Typeable)