Update code formatting and update dependencies (unliftio, protolude) (#152)

* Format code with ormolu

* Use latest unliftio-core

* Use latest protolude
This commit is contained in:
Aditya Manthramurthy 2020-06-14 10:06:41 -07:00 committed by GitHub
parent ce23f7322a
commit 23fecbb469
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
58 changed files with 4875 additions and 4059 deletions

View File

@ -47,7 +47,7 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
cabal v2-update cabal v2-update
cabal v2-build --only-dependencies --enable-tests --enable-benchmarks --verbose=2 cabal v2-build --only-dependencies --enable-tests --enable-benchmarks
- name: Build - name: Build
run: cabal v2-build --enable-tests --enable-benchmarks all run: cabal v2-build --enable-tests --enable-benchmarks all
- name: Run tests - name: Run tests

View File

@ -15,4 +15,5 @@
-- --
import Distribution.Simple import Distribution.Simple
main = defaultMain main = defaultMain

View File

@ -16,11 +16,10 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio
import Control.Monad.IO.Class (liftIO) import Control.Monad.IO.Class (liftIO)
import Network.Minio
import Prelude import Prelude
-- | The following example uses minio's play server at -- | The following example uses minio's play server at
@ -28,8 +27,6 @@ import Prelude
-- credentials are provided via the libary constant, -- credentials are provided via the libary constant,
-- --
-- > minioPlayCI :: ConnectInfo -- > minioPlayCI :: ConnectInfo
--
main :: IO () main :: IO ()
main = do main = do
let bucket = "missingbucket" let bucket = "missingbucket"

View File

@ -16,10 +16,9 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio
import Network.Minio
import UnliftIO.Exception (catch, throwIO) import UnliftIO.Exception (catch, throwIO)
-- | The following example uses minio's play server at -- | The following example uses minio's play server at
@ -27,20 +26,18 @@ import UnliftIO.Exception (catch, throwIO)
-- credentials are provided via the libary constant, -- credentials are provided via the libary constant,
-- --
-- > minioPlayCI :: ConnectInfo -- > minioPlayCI :: ConnectInfo
--
main :: IO () main :: IO ()
main = do main = do
let let bucket = "test"
bucket = "test"
object = "obj" object = "obj"
objectCopy = "obj-copy" objectCopy = "obj-copy"
localFile = "/etc/lsb-release" localFile = "/etc/lsb-release"
res1 <- runMinio minioPlayCI $ do res1 <- runMinio minioPlayCI $ do
-- 1. Make a bucket; Catch BucketAlreadyOwnedByYou exception. -- 1. Make a bucket; Catch BucketAlreadyOwnedByYou exception.
catch (makeBucket bucket Nothing) ( catch
\e -> case e of (makeBucket bucket Nothing)
( \e -> case e of
BucketAlreadyOwnedByYou -> return () BucketAlreadyOwnedByYou -> return ()
_ -> throwIO e _ -> throwIO e
) )
@ -49,8 +46,9 @@ main = do
fPutObject bucket object localFile defaultPutObjectOptions fPutObject bucket object localFile defaultPutObjectOptions
-- 3. Copy bucket/object to bucket/objectCopy. -- 3. Copy bucket/object to bucket/objectCopy.
copyObject defaultDestinationInfo {dstBucket = bucket, dstObject = objectCopy} copyObject
defaultSourceInfo { srcBucket = bucket , srcObject = object } defaultDestinationInfo {dstBucket = bucket, dstObject = objectCopy}
defaultSourceInfo {srcBucket = bucket, srcObject = object}
case res1 of case res1 of
Left e -> putStrLn $ "copyObject failed." ++ show e Left e -> putStrLn $ "copyObject failed." ++ show e

View File

@ -16,18 +16,15 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE ScopedTypeVariables #-}
import Network.Minio
import Data.Monoid ((<>)) import Data.Monoid ((<>))
import Data.Text (pack) import Data.Text (pack)
import Network.Minio
import Options.Applicative import Options.Applicative
import System.FilePath.Posix import System.FilePath.Posix
import UnliftIO (throwIO, try) import UnliftIO (throwIO, try)
import Prelude import Prelude
-- | The following example uses minio's play server at -- | The following example uses minio's play server at
@ -35,21 +32,24 @@ import Prelude
-- credentials are provided via the libary constant, -- credentials are provided via the libary constant,
-- --
-- > minioPlayCI :: ConnectInfo -- > minioPlayCI :: ConnectInfo
--
-- optparse-applicative package based command-line parsing. -- optparse-applicative package based command-line parsing.
fileNameArgs :: Parser FilePath fileNameArgs :: Parser FilePath
fileNameArgs = strArgument fileNameArgs =
(metavar "FILENAME" strArgument
<> help "Name of file to upload to AWS S3 or a MinIO server") ( metavar "FILENAME"
<> help "Name of file to upload to AWS S3 or a MinIO server"
)
cmdParser :: ParserInfo FilePath cmdParser :: ParserInfo FilePath
cmdParser = info cmdParser =
info
(helper <*> fileNameArgs) (helper <*> fileNameArgs)
(fullDesc ( fullDesc
<> progDesc "FileUploader" <> progDesc "FileUploader"
<> header <> header
"FileUploader - a simple file-uploader program using minio-hs") "FileUploader - a simple file-uploader program using minio-hs"
)
main :: IO () main :: IO ()
main = do main = do

View File

@ -16,15 +16,15 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio import Network.Minio
import Network.Minio.AdminAPI import Network.Minio.AdminAPI
import Prelude import Prelude
main :: IO () main :: IO ()
main = do main = do
res <- runMinio minioPlayCI $ res <-
runMinio minioPlayCI $
getConfig getConfig
print res print res

View File

@ -16,13 +16,11 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio
import qualified Data.Conduit as C import qualified Data.Conduit as C
import qualified Data.Conduit.Binary as CB import qualified Data.Conduit.Binary as CB
import Network.Minio
import Prelude import Prelude
-- | The following example uses minio's play server at -- | The following example uses minio's play server at
@ -30,12 +28,9 @@ import Prelude
-- credentials are provided via the libary constant, -- credentials are provided via the libary constant,
-- --
-- > minioPlayCI :: ConnectInfo -- > minioPlayCI :: ConnectInfo
--
main :: IO () main :: IO ()
main = do main = do
let let bucket = "my-bucket"
bucket = "my-bucket"
object = "my-object" object = "my-object"
res <- runMinio minioPlayCI $ do res <- runMinio minioPlayCI $ do
src <- getObject bucket object defaultGetObjectOptions src <- getObject bucket object defaultGetObjectOptions

View File

@ -16,11 +16,10 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio import Network.Minio
import Network.Minio.S3API import Network.Minio.S3API
import Prelude import Prelude
-- | The following example uses minio's play server at -- | The following example uses minio's play server at
@ -28,14 +27,12 @@ import Prelude
-- credentials are provided via the libary constant, -- credentials are provided via the libary constant,
-- --
-- > minioPlayCI :: ConnectInfo -- > minioPlayCI :: ConnectInfo
--
main :: IO () main :: IO ()
main = do main = do
let let bucket = "test"
bucket = "test"
object = "passwd" object = "passwd"
res <- runMinio minioPlayCI $ res <-
runMinio minioPlayCI $
headObject bucket object [] headObject bucket object []
case res of case res of

View File

@ -16,19 +16,23 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio import Network.Minio
import Network.Minio.AdminAPI import Network.Minio.AdminAPI
import Prelude import Prelude
main :: IO () main :: IO ()
main = do main = do
res <- runMinio minioPlayCI $ res <- runMinio minioPlayCI $
do do
hsr <- startHeal Nothing Nothing HealOpts { hoRecursive = True hsr <-
, hoDryRun = False startHeal
Nothing
Nothing
HealOpts
{ hoRecursive = True,
hoDryRun = False
} }
getHealStatus Nothing Nothing (hsrClientToken hsr) getHealStatus Nothing Nothing (hsrClientToken hsr)
print res print res

View File

@ -16,11 +16,10 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio
import Control.Monad.IO.Class (liftIO) import Control.Monad.IO.Class (liftIO)
import Network.Minio
import Prelude import Prelude
-- | The following example uses minio's play server at -- | The following example uses minio's play server at
@ -28,7 +27,6 @@ import Prelude
-- credentials are provided via the libary constant, -- credentials are provided via the libary constant,
-- --
-- > minioPlayCI :: ConnectInfo -- > minioPlayCI :: ConnectInfo
--
-- This example list buckets that belongs to the user and returns -- This example list buckets that belongs to the user and returns
-- region of the first bucket returned. -- region of the first bucket returned.

View File

@ -16,11 +16,10 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio
import Conduit import Conduit
import Network.Minio
import Prelude import Prelude
-- | The following example uses minio's play server at -- | The following example uses minio's play server at
@ -28,20 +27,19 @@ import Prelude
-- credentials are provided via the libary constant, -- credentials are provided via the libary constant,
-- --
-- > minioPlayCI :: ConnectInfo -- > minioPlayCI :: ConnectInfo
--
main :: IO () main :: IO ()
main = do main = do
let let bucket = "test"
bucket = "test"
-- Performs a recursive listing of incomplete uploads under bucket "test" -- Performs a recursive listing of incomplete uploads under bucket "test"
-- on a local minio server. -- on a local minio server.
res <- runMinio minioPlayCI $ res <-
runConduit $ listIncompleteUploads bucket Nothing True .| mapM_C (\v -> (liftIO $ print v)) runMinio minioPlayCI
$ runConduit
$ listIncompleteUploads bucket Nothing True .| mapM_C (\v -> (liftIO $ print v))
print res print res
{- {-
Following is the output of the above program on a local MinIO server. Following is the output of the above program on a local MinIO server.
Right [UploadInfo { uiKey = "go1.6.2.linux-amd64.tar.gz" Right [UploadInfo { uiKey = "go1.6.2.linux-amd64.tar.gz"
@ -50,4 +48,4 @@ main = do
, uiSize = 17731794 , uiSize = 17731794
} }
] ]
-} -}

View File

@ -16,33 +16,31 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio
import Conduit import Conduit
import Network.Minio
import Prelude import Prelude
-- | The following example uses minio's play server at -- | The following example uses minio's play server at
-- https://play.min.io. The endpoint and associated -- https://play.min.io. The endpoint and associated
-- credentials are provided via the libary constant, -- credentials are provided via the libary constant,
-- --
-- > minioPlayCI :: ConnectInfo -- > minioPlayCI :: ConnectInfo
--
main :: IO () main :: IO ()
main = do main = do
let let bucket = "test"
bucket = "test"
-- Performs a recursive listing of all objects under bucket "test" -- Performs a recursive listing of all objects under bucket "test"
-- on play.min.io. -- on play.min.io.
res <- runMinio minioPlayCI $ res <-
runConduit $ listObjects bucket Nothing True .| mapM_C (\v -> (liftIO $ print v)) runMinio minioPlayCI
$ runConduit
$ listObjects bucket Nothing True .| mapM_C (\v -> (liftIO $ print v))
print res print res
{-
{-
Following is the output of the above program on a local MinIO server. Following is the output of the above program on a local MinIO server.
Right [ObjectInfo {oiObject = "ADVANCED.png", oiModTime = 2017-02-10 05:33:24.816 UTC, oiETag = "\"a69f3af6bbb06fe1d42ac910ec30482f\"", oiSize = 94026},ObjectInfo {oiObject = "obj", oiModTime = 2017-02-10 08:49:26.777 UTC, oiETag = "\"715a872a253a3596652c1490081b4b6a-1\"", oiSize = 15728640}] Right [ObjectInfo {oiObject = "ADVANCED.png", oiModTime = 2017-02-10 05:33:24.816 UTC, oiETag = "\"a69f3af6bbb06fe1d42ac910ec30482f\"", oiSize = 94026},ObjectInfo {oiObject = "obj", oiModTime = 2017-02-10 08:49:26.777 UTC, oiETag = "\"715a872a253a3596652c1490081b4b6a-1\"", oiSize = 15728640}]
-} -}

View File

@ -16,11 +16,9 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio
import Network.Minio
import Prelude import Prelude
-- | The following example uses minio's play server at -- | The following example uses minio's play server at
@ -28,12 +26,11 @@ import Prelude
-- credentials are provided via the libary constant, -- credentials are provided via the libary constant,
-- --
-- > minioPlayCI :: ConnectInfo -- > minioPlayCI :: ConnectInfo
--
main :: IO () main :: IO ()
main = do main = do
let bucket = "my-bucket" let bucket = "my-bucket"
res <- runMinio minioPlayCI $ res <-
runMinio minioPlayCI $
-- N B the region provided for makeBucket is optional. -- N B the region provided for makeBucket is optional.
makeBucket bucket (Just "us-east-1") makeBucket bucket (Just "us-east-1")
print res print res

View File

@ -16,34 +16,32 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio
import Control.Monad.IO.Class (liftIO) import Control.Monad.IO.Class (liftIO)
import qualified Data.ByteString.Char8 as B import qualified Data.ByteString.Char8 as B
import Data.CaseInsensitive (original) import Data.CaseInsensitive (original)
import qualified Data.Conduit.Combinators as CC import qualified Data.Conduit.Combinators as CC
import qualified Data.Text.Encoding as E import qualified Data.Text.Encoding as E
import Network.Minio
-- | The following example uses minio's play server at -- | The following example uses minio's play server at
-- https://play.min.io. The endpoint and associated -- https://play.min.io. The endpoint and associated
-- credentials are provided via the libary constant, -- credentials are provided via the libary constant,
-- --
-- > minioPlayCI :: ConnectInfo -- > minioPlayCI :: ConnectInfo
--
main :: IO () main :: IO ()
main = do main = do
let let bucket = "my-bucket"
bucket = "my-bucket"
object = "my-object" object = "my-object"
kb15 = 15*1024 kb15 = 15 * 1024
-- Set query parameter to modify content disposition response -- Set query parameter to modify content disposition response
-- header -- header
queryParam = [("response-content-disposition", queryParam =
Just "attachment; filename=\"your-filename.txt\"")] [ ( "response-content-disposition",
Just "attachment; filename=\"your-filename.txt\""
)
]
res <- runMinio minioPlayCI $ do res <- runMinio minioPlayCI $ do
liftIO $ B.putStrLn "Upload a file that we will fetch with a presigned URL..." liftIO $ B.putStrLn "Upload a file that we will fetch with a presigned URL..."
@ -61,23 +59,28 @@ main = do
-- Generate a URL with 7 days expiry time - note that the headers -- Generate a URL with 7 days expiry time - note that the headers
-- used above must be added to the request with the signed URL -- used above must be added to the request with the signed URL
-- generated. -- generated.
url <- presignedGetObjectUrl "my-bucket" "my-object" (7*24*3600) url <-
queryParam headers presignedGetObjectUrl
"my-bucket"
"my-object"
(7 * 24 * 3600)
queryParam
headers
return (headers, etag, url) return (headers, etag, url)
case res of case res of
Left e -> putStrLn $ "presignedPutObject URL failed." ++ show e Left e -> putStrLn $ "presignedPutObject URL failed." ++ show e
Right (headers, _, url) -> do Right (headers, _, url) -> do
-- We generate a curl command to demonstrate usage of the signed -- We generate a curl command to demonstrate usage of the signed
-- URL. -- URL.
let let hdrOpt (k, v) = B.concat ["-H '", original k, ": ", v, "'"]
hdrOpt (k, v) = B.concat ["-H '", original k, ": ", v, "'"] curlCmd =
curlCmd = B.intercalate " " $ B.intercalate " " $
["curl --fail"] ++ map hdrOpt headers ++ ["curl --fail"] ++ map hdrOpt headers
["-o /tmp/myfile", B.concat ["'", url, "'"]] ++ ["-o /tmp/myfile", B.concat ["'", url, "'"]]
putStrLn $ "The following curl command would use the presigned " ++ putStrLn $
"URL to fetch the object and write it to \"/tmp/myfile\":" "The following curl command would use the presigned "
++ "URL to fetch the object and write it to \"/tmp/myfile\":"
B.putStrLn curlCmd B.putStrLn curlCmd

View File

@ -16,46 +16,42 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio
import qualified Data.ByteString as B import qualified Data.ByteString as B
import qualified Data.ByteString.Char8 as Char8 import qualified Data.ByteString.Char8 as Char8
import qualified Data.HashMap.Strict as H import qualified Data.HashMap.Strict as H
import qualified Data.Text.Encoding as Enc import qualified Data.Text.Encoding as Enc
import qualified Data.Time as Time import qualified Data.Time as Time
import Network.Minio
-- | The following example uses minio's play server at -- | The following example uses minio's play server at
-- https://play.min.io. The endpoint and associated -- https://play.min.io. The endpoint and associated
-- credentials are provided via the libary constant, -- credentials are provided via the libary constant,
-- --
-- > minioPlayCI :: ConnectInfo -- > minioPlayCI :: ConnectInfo
--
main :: IO () main :: IO ()
main = do main = do
now <- Time.getCurrentTime now <- Time.getCurrentTime
let let bucket = "my-bucket"
bucket = "my-bucket"
object = "photos/my-object" object = "photos/my-object"
-- set an expiration time of 10 days -- set an expiration time of 10 days
expireTime = Time.addUTCTime (3600 * 24 * 10) now expireTime = Time.addUTCTime (3600 * 24 * 10) now
-- create a policy with expiration time and conditions - since the -- create a policy with expiration time and conditions - since the
-- conditions are validated, newPostPolicy returns an Either value -- conditions are validated, newPostPolicy returns an Either value
policyE = newPostPolicy expireTime policyE =
newPostPolicy
expireTime
[ -- set the object name condition [ -- set the object name condition
ppCondKey object ppCondKey object,
-- set the bucket name condition -- set the bucket name condition
, ppCondBucket bucket ppCondBucket bucket,
-- set the size range of object as 1B to 10MiB -- set the size range of object as 1B to 10MiB
, ppCondContentLengthRange 1 (10*1024*1024) ppCondContentLengthRange 1 (10 * 1024 * 1024),
-- set content type as jpg image -- set content type as jpg image
, ppCondContentType "image/jpeg" ppCondContentType "image/jpeg",
-- on success set the server response code to 200 -- on success set the server response code to 200
, ppCondSuccessActionStatus 200 ppCondSuccessActionStatus 200
] ]
case policyE of case policyE of
@ -66,12 +62,17 @@ main = do
-- a curl command is output to demonstrate using the generated -- a curl command is output to demonstrate using the generated
-- URL and form-data -- URL and form-data
let let formFn (k, v) =
formFn (k, v) = B.concat ["-F ", Enc.encodeUtf8 k, "=", B.concat
"'", v, "'"] [ "-F ",
Enc.encodeUtf8 k,
"=",
"'",
v,
"'"
]
formOptions = B.intercalate " " $ map formFn $ H.toList formData formOptions = B.intercalate " " $ map formFn $ H.toList formData
return $ B.intercalate " " $ return $ B.intercalate " " $
["curl", formOptions, "-F file=@/tmp/photo.jpg", url] ["curl", formOptions, "-F file=@/tmp/photo.jpg", url]

View File

@ -16,44 +16,42 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio
import qualified Data.ByteString.Char8 as B import qualified Data.ByteString.Char8 as B
import Data.CaseInsensitive (original) import Data.CaseInsensitive (original)
import Network.Minio
-- | The following example uses minio's play server at -- | The following example uses minio's play server at
-- https://play.min.io. The endpoint and associated -- https://play.min.io. The endpoint and associated
-- credentials are provided via the libary constant, -- credentials are provided via the libary constant,
-- --
-- > minioPlayCI :: ConnectInfo -- > minioPlayCI :: ConnectInfo
--
main :: IO () main :: IO ()
main = do main = do
let let -- Use headers to set user-metadata - note that this header will
-- Use headers to set user-metadata - note that this header will
-- need to be set when the URL is used to make an upload. -- need to be set when the URL is used to make an upload.
headers = [("x-amz-meta-url-creator", headers =
"minio-hs-presigned-put-example")] [ ( "x-amz-meta-url-creator",
"minio-hs-presigned-put-example"
)
]
res <- runMinio minioPlayCI $ do res <- runMinio minioPlayCI $ do
-- generate a URL with 7 days expiry time -- generate a URL with 7 days expiry time
presignedPutObjectUrl "my-bucket" "my-object" (7*24*3600) headers presignedPutObjectUrl "my-bucket" "my-object" (7 * 24 * 3600) headers
case res of case res of
Left e -> putStrLn $ "presignedPutObject URL failed." ++ show e Left e -> putStrLn $ "presignedPutObject URL failed." ++ show e
Right url -> do Right url -> do
-- We generate a curl command to demonstrate usage of the signed -- We generate a curl command to demonstrate usage of the signed
-- URL. -- URL.
let let hdrOpt (k, v) = B.concat ["-H '", original k, ": ", v, "'"]
hdrOpt (k, v) = B.concat ["-H '", original k, ": ", v, "'"] curlCmd =
curlCmd = B.intercalate " " $ B.intercalate " " $
["curl "] ++ map hdrOpt headers ++ ["curl "] ++ map hdrOpt headers
["-T /tmp/myfile", B.concat ["'", url, "'"]] ++ ["-T /tmp/myfile", B.concat ["'", url, "'"]]
putStrLn $ "The following curl command would use the presigned " ++ putStrLn $
"URL to upload the file at \"/tmp/myfile\":" "The following curl command would use the presigned "
++ "URL to upload the file at \"/tmp/myfile\":"
B.putStrLn curlCmd B.putStrLn curlCmd

View File

@ -16,12 +16,10 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio
import qualified Data.Conduit.Combinators as CC import qualified Data.Conduit.Combinators as CC
import Network.Minio
import Prelude import Prelude
-- | The following example uses minio's play server at -- | The following example uses minio's play server at
@ -29,25 +27,24 @@ import Prelude
-- credentials are provided via the libary constant, -- credentials are provided via the libary constant,
-- --
-- > minioPlayCI :: ConnectInfo -- > minioPlayCI :: ConnectInfo
--
main :: IO () main :: IO ()
main = do main = do
let let bucket = "test"
bucket = "test"
object = "obj" object = "obj"
localFile = "/etc/lsb-release" localFile = "/etc/lsb-release"
kb15 = 15 * 1024 kb15 = 15 * 1024
-- Eg 1. Upload a stream of repeating "a" using putObject with default options. -- Eg 1. Upload a stream of repeating "a" using putObject with default options.
res1 <- runMinio minioPlayCI $ res1 <-
runMinio minioPlayCI $
putObject bucket object (CC.repeat "a") (Just kb15) defaultPutObjectOptions putObject bucket object (CC.repeat "a") (Just kb15) defaultPutObjectOptions
case res1 of case res1 of
Left e -> putStrLn $ "putObject failed." ++ show e Left e -> putStrLn $ "putObject failed." ++ show e
Right () -> putStrLn "putObject succeeded." Right () -> putStrLn "putObject succeeded."
-- Eg 2. Upload a file using fPutObject with default options. -- Eg 2. Upload a file using fPutObject with default options.
res2 <- runMinio minioPlayCI $ res2 <-
runMinio minioPlayCI $
fPutObject bucket object localFile defaultPutObjectOptions fPutObject bucket object localFile defaultPutObjectOptions
case res2 of case res2 of
Left e -> putStrLn $ "fPutObject failed." ++ show e Left e -> putStrLn $ "fPutObject failed." ++ show e

View File

@ -16,11 +16,9 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio
import Network.Minio
import Prelude import Prelude
-- | The following example uses minio's play server at -- | The following example uses minio's play server at
@ -28,11 +26,8 @@ import Prelude
-- credentials are provided via the libary constant, -- credentials are provided via the libary constant,
-- --
-- > minioPlayCI :: ConnectInfo -- > minioPlayCI :: ConnectInfo
--
main :: IO () main :: IO ()
main = do main = do
let let bucket = "my-bucket"
bucket = "my-bucket"
res <- runMinio minioPlayCI $ removeBucket bucket res <- runMinio minioPlayCI $ removeBucket bucket
print res print res

View File

@ -16,10 +16,9 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio
import Network.Minio
import Prelude import Prelude
-- | The following example uses minio's play server at -- | The following example uses minio's play server at
@ -27,15 +26,13 @@ import Prelude
-- credentials are provided via the libary constant, -- credentials are provided via the libary constant,
-- --
-- > minioPlayCI :: ConnectInfo -- > minioPlayCI :: ConnectInfo
--
main :: IO () main :: IO ()
main = do main = do
let let bucket = "mybucket"
bucket = "mybucket"
object = "myobject" object = "myobject"
res <- runMinio minioPlayCI $ res <-
runMinio minioPlayCI $
removeIncompleteUpload bucket object removeIncompleteUpload bucket object
case res of case res of

View File

@ -16,7 +16,6 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio import Network.Minio
@ -24,11 +23,11 @@ import Prelude
main :: IO () main :: IO ()
main = do main = do
let let bucket = "mybucket"
bucket = "mybucket"
object = "myobject" object = "myobject"
res <- runMinio minioPlayCI $ res <-
runMinio minioPlayCI $
removeObject bucket object removeObject bucket object
case res of case res of

View File

@ -16,26 +16,24 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio
import qualified Conduit as C import qualified Conduit as C
import Control.Monad (when) import Control.Monad (when)
import Network.Minio
import Prelude import Prelude
main :: IO () main :: IO ()
main = do main = do
let bucket = "selectbucket" let bucket = "selectbucket"
object = "1.csv" object = "1.csv"
content = "Name,Place,Temperature\n" content =
"Name,Place,Temperature\n"
<> "James,San Jose,76\n" <> "James,San Jose,76\n"
<> "Alicia,San Leandro,88\n" <> "Alicia,San Leandro,88\n"
<> "Mark,San Carlos,90\n" <> "Mark,San Carlos,90\n"
res <- runMinio minioPlayCI $ do res <- runMinio minioPlayCI $ do
exists <- bucketExists bucket exists <- bucketExists bucket
when (not exists) $ when (not exists) $
makeBucket bucket Nothing makeBucket bucket Nothing

View File

@ -16,15 +16,15 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio import Network.Minio
import Network.Minio.AdminAPI import Network.Minio.AdminAPI
import Prelude import Prelude
main :: IO () main :: IO ()
main = do main = do
res <- runMinio minioPlayCI $ res <-
runMinio minioPlayCI $
getServerInfo getServerInfo
print res print res

View File

@ -16,15 +16,15 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio import Network.Minio
import Network.Minio.AdminAPI import Network.Minio.AdminAPI
import Prelude import Prelude
main :: IO () main :: IO ()
main = do main = do
res <- runMinio minioPlayCI $ res <-
runMinio minioPlayCI $
serviceSendAction ServiceActionRestart serviceSendAction ServiceActionRestart
print res print res

View File

@ -16,15 +16,15 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio import Network.Minio
import Network.Minio.AdminAPI import Network.Minio.AdminAPI
import Prelude import Prelude
main :: IO () main :: IO ()
main = do main = do
res <- runMinio minioPlayCI $ res <-
runMinio minioPlayCI $
serviceSendAction ServiceActionStop serviceSendAction ServiceActionStop
print res print res

View File

@ -16,15 +16,15 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio import Network.Minio
import Network.Minio.AdminAPI import Network.Minio.AdminAPI
import Prelude import Prelude
main :: IO () main :: IO ()
main = do main = do
res <- runMinio minioPlayCI $ res <-
runMinio minioPlayCI $
serviceStatus serviceStatus
print res print res

View File

@ -16,11 +16,10 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
import Network.Minio import Network.Minio
import Network.Minio.AdminAPI import Network.Minio.AdminAPI
import Prelude import Prelude
main :: IO () main :: IO ()

View File

@ -56,7 +56,7 @@ common base-settings
, Network.Minio.XmlParser , Network.Minio.XmlParser
, Network.Minio.JsonParser , Network.Minio.JsonParser
build-depends: base >= 4.7 && < 5 build-depends: base >= 4.7 && < 5
, protolude >= 0.2 && < 0.3 , protolude >= 0.3 && < 0.4
, aeson >= 1.2 , aeson >= 1.2
, base64-bytestring >= 1.0 , base64-bytestring >= 1.0
, binary >= 0.8.5.0 , binary >= 0.8.5.0
@ -83,8 +83,8 @@ common base-settings
, text >= 1.2 , text >= 1.2
, time >= 1.8 , time >= 1.8
, transformers >= 0.5 , transformers >= 0.5
, unliftio >= 0.2 , unliftio >= 0.2 && < 0.3
, unliftio-core >= 0.1 && < 0.2 , unliftio-core >= 0.2 && < 0.3
, unordered-containers >= 0.2 , unordered-containers >= 0.2
, xml-conduit >= 1.8 , xml-conduit >= 1.8

View File

@ -15,19 +15,45 @@
-- --
module Lib.Prelude module Lib.Prelude
( module Exports ( module Exports,
, both both,
) where showBS,
toStrictBS,
import Protolude as Exports hiding (catch, catches, fromStrictBS,
throwIO, try) )
where
import Control.Monad.Trans.Maybe as Exports (MaybeT (..), runMaybeT) import Control.Monad.Trans.Maybe as Exports (MaybeT (..), runMaybeT)
import Data.Time as Exports (UTCTime (..), import qualified Data.ByteString.Lazy as LB
diffUTCTime) import Data.Time as Exports
import UnliftIO as Exports (catch, catches, throwIO, ( UTCTime (..),
try) diffUTCTime,
)
import Protolude as Exports hiding
( Handler,
catch,
catches,
throwIO,
try,
yield,
)
import UnliftIO as Exports
( Handler,
catch,
catches,
throwIO,
try,
)
-- | Apply a function on both elements of a pair -- | Apply a function on both elements of a pair
both :: (a -> b) -> (a, a) -> (b, b) both :: (a -> b) -> (a, a) -> (b, b)
both f (a, b) = (f a, f b) both f (a, b) = (f a, f b)
showBS :: Show a => a -> ByteString
showBS a = toUtf8 (show a :: Text)
toStrictBS :: LByteString -> ByteString
toStrictBS = LB.toStrict
fromStrictBS :: ByteString -> LByteString
fromStrictBS = LB.fromStrict

View File

@ -22,199 +22,200 @@
-- --
-- Types and functions to conveniently access S3 compatible object -- Types and functions to conveniently access S3 compatible object
-- storage servers like MinIO. -- storage servers like MinIO.
module Network.Minio module Network.Minio
( ( -- * Credentials
-- * Credentials Credentials (..),
Credentials (..)
-- ** Credential providers -- ** Credential providers
-- | Run actions that retrieve 'Credentials' from the environment or -- | Run actions that retrieve 'Credentials' from the environment or
-- files or other custom sources. -- files or other custom sources.
, Provider Provider,
, fromAWSConfigFile fromAWSConfigFile,
, fromAWSEnv fromAWSEnv,
, fromMinioEnv fromMinioEnv,
, findFirst findFirst,
-- * Connecting to object storage -- * Connecting to object storage
, ConnectInfo ConnectInfo,
, setRegion setRegion,
, setCreds setCreds,
, setCredsFrom setCredsFrom,
, isConnectInfoSecure isConnectInfoSecure,
, disableTLSCertValidation disableTLSCertValidation,
, MinioConn MinioConn,
, mkMinioConn mkMinioConn,
-- ** Connection helpers -- ** Connection helpers
-- | These are helpers to construct 'ConnectInfo' values for common -- | These are helpers to construct 'ConnectInfo' values for common
-- cases. -- cases.
, minioPlayCI minioPlayCI,
, awsCI awsCI,
, gcsCI gcsCI,
-- * Minio Monad -- * Minio Monad
---------------- ----------------
-- | The Minio Monad provides connection-reuse, bucket-location -- | The Minio Monad provides connection-reuse, bucket-location
-- caching, resource management and simpler error handling -- caching, resource management and simpler error handling
-- functionality. All actions on object storage are performed within -- functionality. All actions on object storage are performed within
-- this Monad. -- this Monad.
, Minio Minio,
, runMinioWith runMinioWith,
, runMinio runMinio,
, runMinioResWith runMinioResWith,
, runMinioRes runMinioRes,
-- * Bucket Operations -- * Bucket Operations
-- ** Creation, removal and querying -- ** Creation, removal and querying
, Bucket Bucket,
, makeBucket makeBucket,
, removeBucket removeBucket,
, bucketExists bucketExists,
, Region Region,
, getLocation getLocation,
-- ** Listing buckets -- ** Listing buckets
, BucketInfo(..) BucketInfo (..),
, listBuckets listBuckets,
-- ** Listing objects -- ** Listing objects
, listObjects listObjects,
, listObjectsV1 listObjectsV1,
, ListItem(..) ListItem (..),
ObjectInfo,
, ObjectInfo oiObject,
, oiObject oiModTime,
, oiModTime oiETag,
, oiETag oiSize,
, oiSize oiUserMetadata,
, oiUserMetadata oiMetadata,
, oiMetadata
-- ** Listing incomplete uploads -- ** Listing incomplete uploads
, listIncompleteUploads listIncompleteUploads,
, UploadId UploadId,
, UploadInfo(..) UploadInfo (..),
, listIncompleteParts listIncompleteParts,
, ObjectPartInfo(..) ObjectPartInfo (..),
-- ** Bucket Notifications -- ** Bucket Notifications
, getBucketNotification getBucketNotification,
, putBucketNotification putBucketNotification,
, removeAllBucketNotification removeAllBucketNotification,
, Notification(..) Notification (..),
, defaultNotification defaultNotification,
, NotificationConfig(..) NotificationConfig (..),
, Arn Arn,
, Event(..) Event (..),
, Filter(..) Filter (..),
, defaultFilter defaultFilter,
, FilterKey(..) FilterKey (..),
, defaultFilterKey defaultFilterKey,
, FilterRules(..) FilterRules (..),
, defaultFilterRules defaultFilterRules,
, FilterRule(..) FilterRule (..),
-- * Object Operations -- * Object Operations
, Object Object,
-- ** File-based operations -- ** File-based operations
, fGetObject fGetObject,
, fPutObject fPutObject,
-- ** Conduit-based streaming operations -- ** Conduit-based streaming operations
, putObject putObject,
, PutObjectOptions PutObjectOptions,
, defaultPutObjectOptions defaultPutObjectOptions,
, pooContentType pooContentType,
, pooContentEncoding pooContentEncoding,
, pooContentDisposition pooContentDisposition,
, pooContentLanguage pooContentLanguage,
, pooCacheControl pooCacheControl,
, pooStorageClass pooStorageClass,
, pooUserMetadata pooUserMetadata,
, pooNumThreads pooNumThreads,
, pooSSE pooSSE,
getObject,
, getObject GetObjectOptions,
, GetObjectOptions defaultGetObjectOptions,
, defaultGetObjectOptions gooRange,
, gooRange gooIfMatch,
, gooIfMatch gooIfNoneMatch,
, gooIfNoneMatch gooIfModifiedSince,
, gooIfModifiedSince gooIfUnmodifiedSince,
, gooIfUnmodifiedSince gooSSECKey,
, gooSSECKey GetObjectResponse,
, GetObjectResponse gorObjectInfo,
, gorObjectInfo gorObjectStream,
, gorObjectStream
-- ** Server-side object copying -- ** Server-side object copying
, copyObject copyObject,
, SourceInfo SourceInfo,
, defaultSourceInfo defaultSourceInfo,
, srcBucket srcBucket,
, srcObject srcObject,
, srcRange srcRange,
, srcIfMatch srcIfMatch,
, srcIfNoneMatch srcIfNoneMatch,
, srcIfModifiedSince srcIfModifiedSince,
, srcIfUnmodifiedSince srcIfUnmodifiedSince,
, DestinationInfo DestinationInfo,
, defaultDestinationInfo defaultDestinationInfo,
, dstBucket dstBucket,
, dstObject dstObject,
-- ** Querying object info -- ** Querying object info
, statObject statObject,
-- ** Object removal operations -- ** Object removal operations
, removeObject removeObject,
, removeIncompleteUpload removeIncompleteUpload,
-- ** Select Object Content with SQL -- ** Select Object Content with SQL
, module Network.Minio.SelectAPI module Network.Minio.SelectAPI,
-- * Server-Side Encryption Helpers -- * Server-Side Encryption Helpers
, mkSSECKey mkSSECKey,
, SSECKey SSECKey,
, SSE(..) SSE (..),
-- * Presigned Operations -- * Presigned Operations
, presignedPutObjectUrl presignedPutObjectUrl,
, presignedGetObjectUrl presignedGetObjectUrl,
, presignedHeadObjectUrl presignedHeadObjectUrl,
, UrlExpiry UrlExpiry,
-- ** POST (browser) upload helpers -- ** POST (browser) upload helpers
-- | Please see -- | Please see
-- https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html -- https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html
-- for detailed information. -- for detailed information.
, newPostPolicy newPostPolicy,
, presignedPostPolicy presignedPostPolicy,
, showPostPolicy showPostPolicy,
, PostPolicy PostPolicy,
, PostPolicyError(..) PostPolicyError (..),
-- *** Post Policy condition helpers -- *** Post Policy condition helpers
, PostPolicyCondition PostPolicyCondition,
, ppCondBucket ppCondBucket,
, ppCondContentLengthRange ppCondContentLengthRange,
, ppCondContentType ppCondContentType,
, ppCondKey ppCondKey,
, ppCondKeyStartsWith ppCondKeyStartsWith,
, ppCondSuccessActionStatus ppCondSuccessActionStatus,
-- * Error handling -- * Error handling
-- | Data types representing various errors that may occur while -- | Data types representing various errors that may occur while
-- working with an object storage service. -- working with an object storage service.
, MinioErr(..) MinioErr (..),
, MErrV(..) MErrV (..),
, ServiceErr(..) ServiceErr (..),
)
) where where
{- {-
This module exports the high-level MinIO API for object storage. This module exports the high-level MinIO API for object storage.
@ -223,9 +224,7 @@ This module exports the high-level MinIO API for object storage.
import qualified Data.Conduit as C import qualified Data.Conduit as C
import qualified Data.Conduit.Binary as CB import qualified Data.Conduit.Binary as CB
import qualified Data.Conduit.Combinators as CC import qualified Data.Conduit.Combinators as CC
import Lib.Prelude import Lib.Prelude
import Network.Minio.CopyObject import Network.Minio.CopyObject
import Network.Minio.Data import Network.Minio.Data
import Network.Minio.Errors import Network.Minio.Errors
@ -248,8 +247,12 @@ fGetObject bucket object fp opts = do
C.connect (gorObjectStream src) $ CB.sinkFileCautious fp C.connect (gorObjectStream src) $ CB.sinkFileCautious fp
-- | Upload the given file to the given object. -- | Upload the given file to the given object.
fPutObject :: Bucket -> Object -> FilePath fPutObject ::
-> PutObjectOptions -> Minio () Bucket ->
Object ->
FilePath ->
PutObjectOptions ->
Minio ()
fPutObject bucket object f opts = fPutObject bucket object f opts =
void $ putObjectInternal bucket object opts $ ODFile f Nothing void $ putObjectInternal bucket object opts $ ODFile f Nothing
@ -257,8 +260,13 @@ fPutObject bucket object f opts =
-- known; this helps the library select optimal part sizes to perform -- known; this helps the library select optimal part sizes to perform
-- a multipart upload. If not specified, it is assumed that the object -- a multipart upload. If not specified, it is assumed that the object
-- can be potentially 5TiB and selects multipart sizes appropriately. -- can be potentially 5TiB and selects multipart sizes appropriately.
putObject :: Bucket -> Object -> C.ConduitM () ByteString Minio () putObject ::
-> Maybe Int64 -> PutObjectOptions -> Minio () Bucket ->
Object ->
C.ConduitM () ByteString Minio () ->
Maybe Int64 ->
PutObjectOptions ->
Minio ()
putObject bucket object src sizeMay opts = putObject bucket object src sizeMay opts =
void $ putObjectInternal bucket object opts $ ODStream src sizeMay void $ putObjectInternal bucket object opts $ ODStream src sizeMay
@ -268,16 +276,23 @@ putObject bucket object src sizeMay opts =
-- copy operation if the new object is to be greater than 5GiB in -- copy operation if the new object is to be greater than 5GiB in
-- size. -- size.
copyObject :: DestinationInfo -> SourceInfo -> Minio () copyObject :: DestinationInfo -> SourceInfo -> Minio ()
copyObject dstInfo srcInfo = void $ copyObjectInternal (dstBucket dstInfo) copyObject dstInfo srcInfo =
(dstObject dstInfo) srcInfo void $
copyObjectInternal
(dstBucket dstInfo)
(dstObject dstInfo)
srcInfo
-- | Remove an object from the object store. -- | Remove an object from the object store.
removeObject :: Bucket -> Object -> Minio () removeObject :: Bucket -> Object -> Minio ()
removeObject = deleteObject removeObject = deleteObject
-- | Get an object from the object store. -- | Get an object from the object store.
getObject :: Bucket -> Object -> GetObjectOptions getObject ::
-> Minio GetObjectResponse Bucket ->
Object ->
GetObjectOptions ->
Minio GetObjectResponse
getObject bucket object opts = getObject bucket object opts =
getObject' bucket object [] $ gooToHeaders opts getObject' bucket object [] $ gooToHeaders opts
@ -309,6 +324,8 @@ bucketExists = headBucket
-- | Removes an ongoing multipart upload of an object. -- | Removes an ongoing multipart upload of an object.
removeIncompleteUpload :: Bucket -> Object -> Minio () removeIncompleteUpload :: Bucket -> Object -> Minio ()
removeIncompleteUpload bucket object = do removeIncompleteUpload bucket object = do
uploads <- C.runConduit $ listIncompleteUploads bucket (Just object) False uploads <-
C.runConduit $
listIncompleteUploads bucket (Just object) False
C..| CC.sinkList C..| CC.sinkList
mapM_ (abortMultipartUpload bucket object) (uiUploadId <$> uploads) mapM_ (abortMultipartUpload bucket object) (uiUploadId <$> uploads)

View File

@ -15,35 +15,35 @@
-- --
module Network.Minio.API module Network.Minio.API
( connect ( connect,
, S3ReqInfo(..) S3ReqInfo (..),
, runMinio runMinio,
, executeRequest executeRequest,
, mkStreamRequest mkStreamRequest,
, getLocation getLocation,
isValidBucketName,
checkBucketNameValidity,
isValidObjectName,
checkObjectNameValidity,
)
where
, isValidBucketName import Control.Retry
, checkBucketNameValidity ( fullJitterBackoff,
, isValidObjectName
, checkObjectNameValidity
) where
import Control.Retry (fullJitterBackoff,
limitRetriesByCumulativeDelay, limitRetriesByCumulativeDelay,
retrying) retrying,
)
import qualified Data.ByteString as B import qualified Data.ByteString as B
import qualified Data.Char as C import qualified Data.Char as C
import qualified Data.Conduit as C import qualified Data.Conduit as C
import qualified Data.HashMap.Strict as H import qualified Data.HashMap.Strict as H
import qualified Data.Text as T import qualified Data.Text as T
import qualified Data.Time.Clock as Time import qualified Data.Time.Clock as Time
import Lib.Prelude
import Network.HTTP.Conduit (Response) import Network.HTTP.Conduit (Response)
import qualified Network.HTTP.Conduit as NC import qualified Network.HTTP.Conduit as NC
import qualified Network.HTTP.Types as HT import qualified Network.HTTP.Types as HT
import Network.HTTP.Types.Header (hHost) import Network.HTTP.Types.Header (hHost)
import Lib.Prelude
import Network.Minio.APICommon import Network.Minio.APICommon
import Network.Minio.Data import Network.Minio.Data
import Network.Minio.Errors import Network.Minio.Errors
@ -54,38 +54,41 @@ import Network.Minio.XmlParser
-- | Fetch bucket location (region) -- | Fetch bucket location (region)
getLocation :: Bucket -> Minio Region getLocation :: Bucket -> Minio Region
getLocation bucket = do getLocation bucket = do
resp <- executeRequest $ defaultS3ReqInfo { resp <-
riBucket = Just bucket executeRequest $
, riQueryParams = [("location", Nothing)] defaultS3ReqInfo
, riNeedsLocation = False { riBucket = Just bucket,
riQueryParams = [("location", Nothing)],
riNeedsLocation = False
} }
parseLocation $ NC.responseBody resp parseLocation $ NC.responseBody resp
-- | Looks for region in RegionMap and updates it using getLocation if -- | Looks for region in RegionMap and updates it using getLocation if
-- absent. -- absent.
discoverRegion :: S3ReqInfo -> Minio (Maybe Region) discoverRegion :: S3ReqInfo -> Minio (Maybe Region)
discoverRegion ri = runMaybeT $ do discoverRegion ri = runMaybeT $ do
bucket <- MaybeT $ return $ riBucket ri bucket <- MaybeT $ return $ riBucket ri
regionMay <- lift $ lookupRegionCache bucket regionMay <- lift $ lookupRegionCache bucket
maybe (do maybe
( do
l <- lift $ getLocation bucket l <- lift $ getLocation bucket
lift $ addToRegionCache bucket l lift $ addToRegionCache bucket l
return l return l
) return regionMay )
return
regionMay
getRegion :: S3ReqInfo -> Minio (Maybe Region) getRegion :: S3ReqInfo -> Minio (Maybe Region)
getRegion ri = do getRegion ri = do
ci <- asks mcConnInfo ci <- asks mcConnInfo
-- getService/makeBucket/getLocation -- don't need location -- getService/makeBucket/getLocation -- don't need location
if | not $ riNeedsLocation ri -> if
| not $ riNeedsLocation ri ->
return $ Just $ connectRegion ci return $ Just $ connectRegion ci
-- if autodiscovery of location is disabled by user -- if autodiscovery of location is disabled by user
| not $ connectAutoDiscoverRegion ci -> | not $ connectAutoDiscoverRegion ci ->
return $ Just $ connectRegion ci return $ Just $ connectRegion ci
-- discover the region for the request -- discover the region for the request
| otherwise -> discoverRegion ri | otherwise -> discoverRegion ri
@ -94,8 +97,11 @@ getRegionHost r = do
ci <- asks mcConnInfo ci <- asks mcConnInfo
if "amazonaws.com" `T.isSuffixOf` connectHost ci if "amazonaws.com" `T.isSuffixOf` connectHost ci
then maybe (throwIO $ MErrVRegionNotSupported r) then
return (H.lookup r awsRegionMap) maybe
(throwIO $ MErrVRegionNotSupported r)
return
(H.lookup r awsRegionMap)
else return $ connectHost ci else return $ connectHost ci
buildRequest :: S3ReqInfo -> Minio NC.Request buildRequest :: S3ReqInfo -> Minio NC.Request
@ -109,27 +115,35 @@ buildRequest ri = do
regionHost <- maybe (return $ connectHost ci) getRegionHost regionMay regionHost <- maybe (return $ connectHost ci) getRegionHost regionMay
let ri' = ri { riHeaders = hostHeader : riHeaders ri let ri' =
, riRegion = regionMay ri
{ riHeaders = hostHeader : riHeaders ri,
riRegion = regionMay
} }
ci' = ci { connectHost = regionHost } ci' = ci {connectHost = regionHost}
hostHeader = (hHost, getHostAddr ci') hostHeader = (hHost, getHostAddr ci')
-- Does not contain body and auth info. -- Does not contain body and auth info.
baseRequest = NC.defaultRequest baseRequest =
{ NC.method = riMethod ri' NC.defaultRequest
, NC.secure = connectIsSecure ci' { NC.method = riMethod ri',
, NC.host = encodeUtf8 $ connectHost ci' NC.secure = connectIsSecure ci',
, NC.port = connectPort ci' NC.host = encodeUtf8 $ connectHost ci',
, NC.path = getS3Path (riBucket ri') (riObject ri') NC.port = connectPort ci',
, NC.requestHeaders = riHeaders ri' NC.path = getS3Path (riBucket ri') (riObject ri'),
, NC.queryString = HT.renderQuery False $ riQueryParams ri' NC.requestHeaders = riHeaders ri',
NC.queryString = HT.renderQuery False $ riQueryParams ri'
} }
timeStamp <- liftIO Time.getCurrentTime timeStamp <- liftIO Time.getCurrentTime
let sp = SignParams (connectAccessKey ci') (connectSecretKey ci') let sp =
timeStamp (riRegion ri') Nothing Nothing SignParams
(connectAccessKey ci')
(connectSecretKey ci')
timeStamp
(riRegion ri')
Nothing
Nothing
-- Cases to handle: -- Cases to handle:
-- --
@ -142,42 +156,46 @@ buildRequest ri = do
-- sha256hash, buffer request in memory and perform request. -- sha256hash, buffer request in memory and perform request.
-- case 2 from above. -- case 2 from above.
if | isStreamingPayload (riPayload ri') && if
(not $ connectIsSecure ci') -> do | isStreamingPayload (riPayload ri')
&& (not $ connectIsSecure ci') -> do
(pLen, pSrc) <- case riPayload ri of (pLen, pSrc) <- case riPayload ri of
PayloadC l src -> return (l, src) PayloadC l src -> return (l, src)
_ -> throwIO MErrVUnexpectedPayload _ -> throwIO MErrVUnexpectedPayload
let reqFn = signV4Stream pLen sp baseRequest let reqFn = signV4Stream pLen sp baseRequest
return $ reqFn pSrc return $ reqFn pSrc
| otherwise -> do | otherwise -> do
-- case 1 described above. -- case 1 described above.
sp' <- if | connectIsSecure ci' -> return sp sp' <-
if
| connectIsSecure ci' -> return sp
-- case 3 described above. -- case 3 described above.
| otherwise -> do | otherwise -> do
pHash <- getPayloadSHA256Hash $ riPayload ri' pHash <- getPayloadSHA256Hash $ riPayload ri'
return $ sp { spPayloadHash = Just pHash } return $ sp {spPayloadHash = Just pHash}
let signHeaders = signV4 sp' baseRequest let signHeaders = signV4 sp' baseRequest
return $ baseRequest return $
baseRequest
{ NC.requestHeaders = { NC.requestHeaders =
NC.requestHeaders baseRequest ++ NC.requestHeaders baseRequest
mkHeaderFromPairs signHeaders ++ mkHeaderFromPairs signHeaders,
, NC.requestBody = getRequestBody (riPayload ri') NC.requestBody = getRequestBody (riPayload ri')
} }
retryAPIRequest :: Minio a -> Minio a retryAPIRequest :: Minio a -> Minio a
retryAPIRequest apiCall = do retryAPIRequest apiCall = do
resE <- retrying retryPolicy (const shouldRetry) $ resE <-
const $ try apiCall retrying retryPolicy (const shouldRetry)
$ const
$ try apiCall
either throwIO return resE either throwIO return resE
where where
-- Retry using the full-jitter backoff method for up to 10 mins -- Retry using the full-jitter backoff method for up to 10 mins
-- total -- total
retryPolicy = limitRetriesByCumulativeDelay tenMins retryPolicy =
$ fullJitterBackoff oneMilliSecond limitRetriesByCumulativeDelay tenMins $
fullJitterBackoff oneMilliSecond
oneMilliSecond = 1000 -- in microseconds oneMilliSecond = 1000 -- in microseconds
tenMins = 10 * 60 * 1000000 -- in microseconds tenMins = 10 * 60 * 1000000 -- in microseconds
-- retry on connection related failure -- retry on connection related failure
@ -197,15 +215,15 @@ retryAPIRequest apiCall = do
-- We received an unexpected exception -- We received an unexpected exception
Left someOtherExn -> throwIO someOtherExn Left someOtherExn -> throwIO someOtherExn
executeRequest :: S3ReqInfo -> Minio (Response LByteString) executeRequest :: S3ReqInfo -> Minio (Response LByteString)
executeRequest ri = do executeRequest ri = do
req <- buildRequest ri req <- buildRequest ri
mgr <- asks mcConnManager mgr <- asks mcConnManager
retryAPIRequest $ httpLbs req mgr retryAPIRequest $ httpLbs req mgr
mkStreamRequest :: S3ReqInfo mkStreamRequest ::
-> Minio (Response (C.ConduitM () ByteString Minio ())) S3ReqInfo ->
Minio (Response (C.ConduitM () ByteString Minio ()))
mkStreamRequest ri = do mkStreamRequest ri = do
req <- buildRequest ri req <- buildRequest ri
mgr <- asks mcConnManager mgr <- asks mcConnManager
@ -214,35 +232,43 @@ mkStreamRequest ri = do
-- Bucket name validity check according to AWS rules. -- Bucket name validity check according to AWS rules.
isValidBucketName :: Bucket -> Bool isValidBucketName :: Bucket -> Bool
isValidBucketName bucket = isValidBucketName bucket =
not (or [ len < 3 || len > 63 not
, or (map labelCheck labels) ( or
, or (map labelCharsCheck labels) [ len < 3 || len > 63,
, isIPCheck or (map labelCheck labels),
]) or (map labelCharsCheck labels),
isIPCheck
]
)
where where
len = T.length bucket len = T.length bucket
labels = T.splitOn "." bucket labels = T.splitOn "." bucket
-- does label `l` fail basic checks of length and start/end? -- does label `l` fail basic checks of length and start/end?
labelCheck l = T.length l == 0 || T.head l == '-' || T.last l == '-' labelCheck l = T.length l == 0 || T.head l == '-' || T.last l == '-'
-- does label `l` have non-allowed characters? -- does label `l` have non-allowed characters?
labelCharsCheck l = isJust $ T.find (\x -> not (C.isAsciiLower x || labelCharsCheck l =
x == '-' || isJust $
C.isDigit x)) l T.find
( \x ->
not
( C.isAsciiLower x
|| x == '-'
|| C.isDigit x
)
)
l
-- does label `l` have non-digit characters? -- does label `l` have non-digit characters?
labelNonDigits l = isJust $ T.find (not . C.isDigit) l labelNonDigits l = isJust $ T.find (not . C.isDigit) l
labelAsNums = map (not . labelNonDigits) labels labelAsNums = map (not . labelNonDigits) labels
-- check if bucket name looks like an IP -- check if bucket name looks like an IP
isIPCheck = and labelAsNums && length labelAsNums == 4 isIPCheck = and labelAsNums && length labelAsNums == 4
-- Throws exception iff bucket name is invalid according to AWS rules. -- Throws exception iff bucket name is invalid according to AWS rules.
checkBucketNameValidity :: MonadIO m => Bucket -> m () checkBucketNameValidity :: MonadIO m => Bucket -> m ()
checkBucketNameValidity bucket = checkBucketNameValidity bucket =
when (not $ isValidBucketName bucket) $ when (not $ isValidBucketName bucket)
throwIO $ MErrVInvalidBucketName bucket $ throwIO
$ MErrVInvalidBucketName bucket
isValidObjectName :: Object -> Bool isValidObjectName :: Object -> Bool
isValidObjectName object = isValidObjectName object =
@ -250,5 +276,6 @@ isValidObjectName object =
checkObjectNameValidity :: MonadIO m => Object -> m () checkObjectNameValidity :: MonadIO m => Object -> m ()
checkObjectNameValidity object = checkObjectNameValidity object =
when (not $ isValidObjectName object) $ when (not $ isValidObjectName object)
throwIO $ MErrVInvalidObjectName object $ throwIO
$ MErrVInvalidObjectName object

View File

@ -20,24 +20,24 @@ import qualified Conduit as C
import qualified Data.ByteString as BS import qualified Data.ByteString as BS
import qualified Data.ByteString.Lazy as LB import qualified Data.ByteString.Lazy as LB
import Data.Conduit.Binary (sourceHandleRange) import Data.Conduit.Binary (sourceHandleRange)
import Lib.Prelude
import qualified Network.HTTP.Conduit as NC import qualified Network.HTTP.Conduit as NC
import qualified Network.HTTP.Types as HT import qualified Network.HTTP.Types as HT
import Lib.Prelude
import Network.Minio.Data import Network.Minio.Data
import Network.Minio.Data.Crypto import Network.Minio.Data.Crypto
import Network.Minio.Errors import Network.Minio.Errors
sha256Header :: ByteString -> HT.Header sha256Header :: ByteString -> HT.Header
sha256Header = ("x-amz-content-sha256", ) sha256Header = ("x-amz-content-sha256",)
-- | This function throws an error if the payload is a conduit (as it -- | This function throws an error if the payload is a conduit (as it
-- will not be possible to re-read the conduit after it is consumed). -- will not be possible to re-read the conduit after it is consumed).
getPayloadSHA256Hash :: Payload -> Minio ByteString getPayloadSHA256Hash :: Payload -> Minio ByteString
getPayloadSHA256Hash (PayloadBS bs) = return $ hashSHA256 bs getPayloadSHA256Hash (PayloadBS bs) = return $ hashSHA256 bs
getPayloadSHA256Hash (PayloadH h off size) = hashSHA256FromSource $ getPayloadSHA256Hash (PayloadH h off size) =
sourceHandleRange h hashSHA256FromSource $
sourceHandleRange
h
(return . fromIntegral $ off) (return . fromIntegral $ off)
(return . fromIntegral $ size) (return . fromIntegral $ size)
getPayloadSHA256Hash (PayloadC _ _) = throwIO MErrVUnexpectedPayload getPayloadSHA256Hash (PayloadC _ _) = throwIO MErrVUnexpectedPayload
@ -46,7 +46,8 @@ getRequestBody :: Payload -> NC.RequestBody
getRequestBody (PayloadBS bs) = NC.RequestBodyBS bs getRequestBody (PayloadBS bs) = NC.RequestBodyBS bs
getRequestBody (PayloadH h off size) = getRequestBody (PayloadH h off size) =
NC.requestBodySource (fromIntegral size) $ NC.requestBodySource (fromIntegral size) $
sourceHandleRange h sourceHandleRange
h
(return . fromIntegral $ off) (return . fromIntegral $ off)
(return . fromIntegral $ size) (return . fromIntegral $ size)
getRequestBody (PayloadC n src) = NC.requestBodySource n src getRequestBody (PayloadC n src) = NC.requestBodySource n src
@ -55,10 +56,13 @@ mkStreamingPayload :: Payload -> Payload
mkStreamingPayload payload = mkStreamingPayload payload =
case payload of case payload of
PayloadBS bs -> PayloadBS bs ->
PayloadC (fromIntegral $ BS.length bs) PayloadC
(fromIntegral $ BS.length bs)
(C.sourceLazy $ LB.fromStrict bs) (C.sourceLazy $ LB.fromStrict bs)
PayloadH h off len -> PayloadH h off len ->
PayloadC len $ sourceHandleRange h PayloadC len $
sourceHandleRange
h
(return . fromIntegral $ off) (return . fromIntegral $ off)
(return . fromIntegral $ len) (return . fromIntegral $ len)
_ -> payload _ -> payload

View File

@ -17,60 +17,67 @@
module Network.Minio.AdminAPI module Network.Minio.AdminAPI
( -- * MinIO Admin API ( -- * MinIO Admin API
-------------------- --------------------
-- | Provides MinIO admin API and related types. It is in -- | Provides MinIO admin API and related types. It is in
-- experimental state. -- experimental state.
DriveInfo(..) DriveInfo (..),
, ErasureInfo(..) ErasureInfo (..),
, Backend(..) Backend (..),
, ConnStats(..) ConnStats (..),
, HttpStats(..) HttpStats (..),
, ServerProps(..) ServerProps (..),
, CountNAvgTime(..) CountNAvgTime (..),
, StorageClass(..) StorageClass (..),
, StorageInfo(..) StorageInfo (..),
, SIData(..) SIData (..),
, ServerInfo(..) ServerInfo (..),
, getServerInfo getServerInfo,
HealOpts (..),
HealResultItem (..),
HealStatus (..),
HealStartResp (..),
startHeal,
forceStartHeal,
getHealStatus,
SetConfigResult (..),
NodeSummary (..),
setConfig,
getConfig,
ServerVersion (..),
ServiceStatus (..),
serviceStatus,
ServiceAction (..),
serviceSendAction,
)
where
, HealOpts(..) import Data.Aeson
, HealResultItem(..) ( (.:),
, HealStatus(..) (.:?),
, HealStartResp(..) (.=),
, startHeal FromJSON,
, forceStartHeal ToJSON,
, getHealStatus Value (Object),
eitherDecode,
, SetConfigResult(..) object,
, NodeSummary(..) pairs,
, setConfig parseJSON,
, getConfig toEncoding,
toJSON,
, ServerVersion(..) withObject,
, ServiceStatus(..) withText,
, serviceStatus )
, ServiceAction(..)
, serviceSendAction
) where
import Data.Aeson (FromJSON, ToJSON, Value (Object),
eitherDecode, object, pairs,
parseJSON, toEncoding, toJSON,
withObject, withText, (.:), (.:?),
(.=))
import qualified Data.Aeson as A import qualified Data.Aeson as A
import Data.Aeson.Types (typeMismatch) import Data.Aeson.Types (typeMismatch)
import qualified Data.ByteString as B import qualified Data.ByteString as B
import qualified Data.ByteString.Lazy as LBS import qualified Data.ByteString.Lazy as LBS
import qualified Data.Text as T import qualified Data.Text as T
import Data.Time (NominalDiffTime, getCurrentTime) import Data.Time (NominalDiffTime, getCurrentTime)
import Lib.Prelude
import Network.HTTP.Conduit (Response) import Network.HTTP.Conduit (Response)
import qualified Network.HTTP.Conduit as NC import qualified Network.HTTP.Conduit as NC
import qualified Network.HTTP.Types as HT import qualified Network.HTTP.Types as HT
import Network.HTTP.Types.Header (hHost) import Network.HTTP.Types.Header (hHost)
import Lib.Prelude
import Network.Minio.APICommon import Network.Minio.APICommon
import Network.Minio.Data import Network.Minio.Data
import Network.Minio.Errors import Network.Minio.Errors
@ -78,44 +85,51 @@ import Network.Minio.Sign.V4
import Network.Minio.Utils import Network.Minio.Utils
data DriveInfo = DriveInfo data DriveInfo = DriveInfo
{ diUuid :: Text { diUuid :: Text,
, diEndpoint :: Text diEndpoint :: Text,
, diState :: Text diState :: Text
} deriving (Eq, Show) }
deriving (Eq, Show)
instance FromJSON DriveInfo where instance FromJSON DriveInfo where
parseJSON = withObject "DriveInfo" $ \v -> DriveInfo parseJSON = withObject "DriveInfo" $ \v ->
DriveInfo
<$> v .: "uuid" <$> v .: "uuid"
<*> v .: "endpoint" <*> v .: "endpoint"
<*> v .: "state" <*> v .: "state"
data StorageClass = StorageClass data StorageClass = StorageClass
{ scParity :: Int { scParity :: Int,
, scData :: Int scData :: Int
} deriving (Eq, Show) }
deriving (Eq, Show)
data ErasureInfo = ErasureInfo data ErasureInfo = ErasureInfo
{ eiOnlineDisks :: Int { eiOnlineDisks :: Int,
, eiOfflineDisks :: Int eiOfflineDisks :: Int,
, eiStandard :: StorageClass eiStandard :: StorageClass,
, eiReducedRedundancy :: StorageClass eiReducedRedundancy :: StorageClass,
, eiSets :: [[DriveInfo]] eiSets :: [[DriveInfo]]
} deriving (Eq, Show) }
deriving (Eq, Show)
instance FromJSON ErasureInfo where instance FromJSON ErasureInfo where
parseJSON = withObject "ErasureInfo" $ \v -> do parseJSON = withObject "ErasureInfo" $ \v -> do
onlineDisks <- v .: "OnlineDisks" onlineDisks <- v .: "OnlineDisks"
offlineDisks <- v .: "OfflineDisks" offlineDisks <- v .: "OfflineDisks"
stdClass <- StorageClass stdClass <-
StorageClass
<$> v .: "StandardSCData" <$> v .: "StandardSCData"
<*> v .: "StandardSCParity" <*> v .: "StandardSCParity"
rrClass <- StorageClass rrClass <-
StorageClass
<$> v .: "RRSCData" <$> v .: "RRSCData"
<*> v .: "RRSCParity" <*> v .: "RRSCParity"
sets <- v .: "Sets" sets <- v .: "Sets"
return $ ErasureInfo onlineDisks offlineDisks stdClass rrClass sets return $ ErasureInfo onlineDisks offlineDisks stdClass rrClass sets
data Backend = BackendFS data Backend
= BackendFS
| BackendErasure ErasureInfo | BackendErasure ErasureInfo
deriving (Eq, Show) deriving (Eq, Show)
@ -128,22 +142,25 @@ instance FromJSON Backend where
_ -> typeMismatch "BackendType" (Object v) _ -> typeMismatch "BackendType" (Object v)
data ConnStats = ConnStats data ConnStats = ConnStats
{ csTransferred :: Int64 { csTransferred :: Int64,
, csReceived :: Int64 csReceived :: Int64
} deriving (Eq, Show) }
deriving (Eq, Show)
instance FromJSON ConnStats where instance FromJSON ConnStats where
parseJSON = withObject "ConnStats" $ \v -> ConnStats parseJSON = withObject "ConnStats" $ \v ->
ConnStats
<$> v .: "transferred" <$> v .: "transferred"
<*> v .: "received" <*> v .: "received"
data ServerProps = ServerProps data ServerProps = ServerProps
{ spUptime :: NominalDiffTime { spUptime :: NominalDiffTime,
, spVersion :: Text spVersion :: Text,
, spCommitId :: Text spCommitId :: Text,
, spRegion :: Text spRegion :: Text,
, spSqsArns :: [Text] spSqsArns :: [Text]
} deriving (Eq, Show) }
deriving (Eq, Show)
instance FromJSON ServerProps where instance FromJSON ServerProps where
parseJSON = withObject "SIServer" $ \v -> do parseJSON = withObject "SIServer" $ \v -> do
@ -156,40 +173,46 @@ instance FromJSON ServerProps where
return $ ServerProps uptime ver commitId region arn return $ ServerProps uptime ver commitId region arn
data StorageInfo = StorageInfo data StorageInfo = StorageInfo
{ siUsed :: Int64 { siUsed :: Int64,
, siBackend :: Backend siBackend :: Backend
} deriving (Eq, Show) }
deriving (Eq, Show)
instance FromJSON StorageInfo where instance FromJSON StorageInfo where
parseJSON = withObject "StorageInfo" $ \v -> StorageInfo parseJSON = withObject "StorageInfo" $ \v ->
StorageInfo
<$> v .: "Used" <$> v .: "Used"
<*> v .: "Backend" <*> v .: "Backend"
data CountNAvgTime = CountNAvgTime data CountNAvgTime = CountNAvgTime
{ caCount :: Int64 { caCount :: Int64,
, caAvgDuration :: Text caAvgDuration :: Text
} deriving (Eq, Show) }
deriving (Eq, Show)
instance FromJSON CountNAvgTime where instance FromJSON CountNAvgTime where
parseJSON = withObject "CountNAvgTime" $ \v -> CountNAvgTime parseJSON = withObject "CountNAvgTime" $ \v ->
CountNAvgTime
<$> v .: "count" <$> v .: "count"
<*> v .: "avgDuration" <*> v .: "avgDuration"
data HttpStats = HttpStats data HttpStats = HttpStats
{ hsTotalHeads :: CountNAvgTime { hsTotalHeads :: CountNAvgTime,
, hsSuccessHeads :: CountNAvgTime hsSuccessHeads :: CountNAvgTime,
, hsTotalGets :: CountNAvgTime hsTotalGets :: CountNAvgTime,
, hsSuccessGets :: CountNAvgTime hsSuccessGets :: CountNAvgTime,
, hsTotalPuts :: CountNAvgTime hsTotalPuts :: CountNAvgTime,
, hsSuccessPuts :: CountNAvgTime hsSuccessPuts :: CountNAvgTime,
, hsTotalPosts :: CountNAvgTime hsTotalPosts :: CountNAvgTime,
, hsSuccessPosts :: CountNAvgTime hsSuccessPosts :: CountNAvgTime,
, hsTotalDeletes :: CountNAvgTime hsTotalDeletes :: CountNAvgTime,
, hsSuccessDeletes :: CountNAvgTime hsSuccessDeletes :: CountNAvgTime
} deriving (Eq, Show) }
deriving (Eq, Show)
instance FromJSON HttpStats where instance FromJSON HttpStats where
parseJSON = withObject "HttpStats" $ \v -> HttpStats parseJSON = withObject "HttpStats" $ \v ->
HttpStats
<$> v .: "totalHEADs" <$> v .: "totalHEADs"
<*> v .: "successHEADs" <*> v .: "successHEADs"
<*> v .: "totalGETs" <*> v .: "totalGETs"
@ -202,45 +225,52 @@ instance FromJSON HttpStats where
<*> v .: "successDELETEs" <*> v .: "successDELETEs"
data SIData = SIData data SIData = SIData
{ sdStorage :: StorageInfo { sdStorage :: StorageInfo,
, sdConnStats :: ConnStats sdConnStats :: ConnStats,
, sdHttpStats :: HttpStats sdHttpStats :: HttpStats,
, sdProps :: ServerProps sdProps :: ServerProps
} deriving (Eq, Show) }
deriving (Eq, Show)
instance FromJSON SIData where instance FromJSON SIData where
parseJSON = withObject "SIData" $ \v -> SIData parseJSON = withObject "SIData" $ \v ->
SIData
<$> v .: "storage" <$> v .: "storage"
<*> v .: "network" <*> v .: "network"
<*> v .: "http" <*> v .: "http"
<*> v .: "server" <*> v .: "server"
data ServerInfo = ServerInfo data ServerInfo = ServerInfo
{ siError :: Text { siError :: Text,
, siAddr :: Text siAddr :: Text,
, siData :: SIData siData :: SIData
} deriving (Eq, Show) }
deriving (Eq, Show)
instance FromJSON ServerInfo where instance FromJSON ServerInfo where
parseJSON = withObject "ServerInfo" $ \v -> ServerInfo parseJSON = withObject "ServerInfo" $ \v ->
ServerInfo
<$> v .: "error" <$> v .: "error"
<*> v .: "addr" <*> v .: "addr"
<*> v .: "data" <*> v .: "data"
data ServerVersion = ServerVersion data ServerVersion = ServerVersion
{ svVersion :: Text { svVersion :: Text,
, svCommitId :: Text svCommitId :: Text
} deriving (Eq, Show) }
deriving (Eq, Show)
instance FromJSON ServerVersion where instance FromJSON ServerVersion where
parseJSON = withObject "ServerVersion" $ \v -> ServerVersion parseJSON = withObject "ServerVersion" $ \v ->
ServerVersion
<$> v .: "version" <$> v .: "version"
<*> v .: "commitID" <*> v .: "commitID"
data ServiceStatus = ServiceStatus data ServiceStatus = ServiceStatus
{ ssVersion :: ServerVersion { ssVersion :: ServerVersion,
, ssUptime :: NominalDiffTime ssUptime :: NominalDiffTime
} deriving (Eq, Show) }
deriving (Eq, Show)
instance FromJSON ServiceStatus where instance FromJSON ServiceStatus where
parseJSON = withObject "ServiceStatus" $ \v -> do parseJSON = withObject "ServiceStatus" $ \v -> do
@ -249,12 +279,13 @@ instance FromJSON ServiceStatus where
let uptime = uptimeNs / 1e9 let uptime = uptimeNs / 1e9
return $ ServiceStatus serverVersion uptime return $ ServiceStatus serverVersion uptime
data ServiceAction = ServiceActionRestart data ServiceAction
= ServiceActionRestart
| ServiceActionStop | ServiceActionStop
deriving (Eq, Show) deriving (Eq, Show)
instance ToJSON ServiceAction where instance ToJSON ServiceAction where
toJSON a = object [ "action" .= serviceActionToText a ] toJSON a = object ["action" .= serviceActionToText a]
serviceActionToText :: ServiceAction -> Text serviceActionToText :: ServiceAction -> Text
serviceActionToText a = case a of serviceActionToText a = case a of
@ -265,21 +296,24 @@ adminPath :: ByteString
adminPath = "/minio/admin" adminPath = "/minio/admin"
data HealStartResp = HealStartResp data HealStartResp = HealStartResp
{ hsrClientToken :: Text { hsrClientToken :: Text,
, hsrClientAddr :: Text hsrClientAddr :: Text,
, hsrStartTime :: UTCTime hsrStartTime :: UTCTime
} deriving (Eq, Show) }
deriving (Eq, Show)
instance FromJSON HealStartResp where instance FromJSON HealStartResp where
parseJSON = withObject "HealStartResp" $ \v -> HealStartResp parseJSON = withObject "HealStartResp" $ \v ->
HealStartResp
<$> v .: "clientToken" <$> v .: "clientToken"
<*> v .: "clientAddress" <*> v .: "clientAddress"
<*> v .: "startTime" <*> v .: "startTime"
data HealOpts = HealOpts data HealOpts = HealOpts
{ hoRecursive :: Bool { hoRecursive :: Bool,
, hoDryRun :: Bool hoDryRun :: Bool
} deriving (Eq, Show) }
deriving (Eq, Show)
instance ToJSON HealOpts where instance ToJSON HealOpts where
toJSON (HealOpts r d) = toJSON (HealOpts r d) =
@ -288,11 +322,13 @@ instance ToJSON HealOpts where
pairs ("recursive" .= r <> "dryRun" .= d) pairs ("recursive" .= r <> "dryRun" .= d)
instance FromJSON HealOpts where instance FromJSON HealOpts where
parseJSON = withObject "HealOpts" $ \v -> HealOpts parseJSON = withObject "HealOpts" $ \v ->
HealOpts
<$> v .: "recursive" <$> v .: "recursive"
<*> v .: "dryRun" <*> v .: "dryRun"
data HealItemType = HealItemMetadata data HealItemType
= HealItemMetadata
| HealItemBucket | HealItemBucket
| HealItemBucketMetadata | HealItemBucketMetadata
| HealItemObject | HealItemObject
@ -307,44 +343,50 @@ instance FromJSON HealItemType where
_ -> typeMismatch "HealItemType" (A.String v) _ -> typeMismatch "HealItemType" (A.String v)
data NodeSummary = NodeSummary data NodeSummary = NodeSummary
{ nsName :: Text { nsName :: Text,
, nsErrSet :: Bool nsErrSet :: Bool,
, nsErrMessage :: Text nsErrMessage :: Text
} deriving (Eq, Show) }
deriving (Eq, Show)
instance FromJSON NodeSummary where instance FromJSON NodeSummary where
parseJSON = withObject "NodeSummary" $ \v -> NodeSummary parseJSON = withObject "NodeSummary" $ \v ->
NodeSummary
<$> v .: "name" <$> v .: "name"
<*> v .: "errSet" <*> v .: "errSet"
<*> v .: "errMsg" <*> v .: "errMsg"
data SetConfigResult = SetConfigResult data SetConfigResult = SetConfigResult
{ scrStatus :: Bool { scrStatus :: Bool,
, scrNodeSummary :: [NodeSummary] scrNodeSummary :: [NodeSummary]
} deriving (Eq, Show) }
deriving (Eq, Show)
instance FromJSON SetConfigResult where instance FromJSON SetConfigResult where
parseJSON = withObject "SetConfigResult" $ \v -> SetConfigResult parseJSON = withObject "SetConfigResult" $ \v ->
SetConfigResult
<$> v .: "status" <$> v .: "status"
<*> v .: "nodeResults" <*> v .: "nodeResults"
data HealResultItem = HealResultItem data HealResultItem = HealResultItem
{ hriResultIdx :: Int { hriResultIdx :: Int,
, hriType :: HealItemType hriType :: HealItemType,
, hriBucket :: Bucket hriBucket :: Bucket,
, hriObject :: Object hriObject :: Object,
, hriDetail :: Text hriDetail :: Text,
, hriParityBlocks :: Maybe Int hriParityBlocks :: Maybe Int,
, hriDataBlocks :: Maybe Int hriDataBlocks :: Maybe Int,
, hriDiskCount :: Int hriDiskCount :: Int,
, hriSetCount :: Int hriSetCount :: Int,
, hriObjectSize :: Int hriObjectSize :: Int,
, hriBefore :: [DriveInfo] hriBefore :: [DriveInfo],
, hriAfter :: [DriveInfo] hriAfter :: [DriveInfo]
} deriving (Eq, Show) }
deriving (Eq, Show)
instance FromJSON HealResultItem where instance FromJSON HealResultItem where
parseJSON = withObject "HealResultItem" $ \v -> HealResultItem parseJSON = withObject "HealResultItem" $ \v ->
HealResultItem
<$> v .: "resultId" <$> v .: "resultId"
<*> v .: "type" <*> v .: "type"
<*> v .: "bucket" <*> v .: "bucket"
@ -355,22 +397,28 @@ instance FromJSON HealResultItem where
<*> v .: "diskCount" <*> v .: "diskCount"
<*> v .: "setCount" <*> v .: "setCount"
<*> v .: "objectSize" <*> v .: "objectSize"
<*> (do before <- v .: "before" <*> ( do
before .: "drives") before <- v .: "before"
<*> (do after <- v .: "after" before .: "drives"
after .: "drives") )
<*> ( do
after <- v .: "after"
after .: "drives"
)
data HealStatus = HealStatus data HealStatus = HealStatus
{ hsSummary :: Text { hsSummary :: Text,
, hsStartTime :: UTCTime hsStartTime :: UTCTime,
, hsSettings :: HealOpts hsSettings :: HealOpts,
, hsNumDisks :: Int hsNumDisks :: Int,
, hsFailureDetail :: Maybe Text hsFailureDetail :: Maybe Text,
, hsItems :: Maybe [HealResultItem] hsItems :: Maybe [HealResultItem]
} deriving (Eq, Show) }
deriving (Eq, Show)
instance FromJSON HealStatus where instance FromJSON HealStatus where
parseJSON = withObject "HealStatus" $ \v -> HealStatus parseJSON = withObject "HealStatus" $ \v ->
HealStatus
<$> v .: "Summary" <$> v .: "Summary"
<*> v .: "StartTime" <*> v .: "StartTime"
<*> v .: "Settings" <*> v .: "Settings"
@ -381,19 +429,24 @@ instance FromJSON HealStatus where
healPath :: Maybe Bucket -> Maybe Text -> ByteString healPath :: Maybe Bucket -> Maybe Text -> ByteString
healPath bucket prefix = do healPath bucket prefix = do
if (isJust bucket) if (isJust bucket)
then encodeUtf8 $ "v1/heal/" <> fromMaybe "" bucket <> "/" then
encodeUtf8 $
"v1/heal/" <> fromMaybe "" bucket <> "/"
<> fromMaybe "" prefix <> fromMaybe "" prefix
else encodeUtf8 $ "v1/heal/" else encodeUtf8 $ "v1/heal/"
-- | Get server version and uptime. -- | Get server version and uptime.
serviceStatus :: Minio ServiceStatus serviceStatus :: Minio ServiceStatus
serviceStatus = do serviceStatus = do
rsp <- executeAdminRequest AdminReqInfo { ariMethod = HT.methodGet rsp <-
, ariPayload = PayloadBS B.empty executeAdminRequest
, ariPayloadHash = Nothing AdminReqInfo
, ariPath = "v1/service" { ariMethod = HT.methodGet,
, ariHeaders = [] ariPayload = PayloadBS B.empty,
, ariQueryParams = [] ariPayloadHash = Nothing,
ariPath = "v1/service",
ariHeaders = [],
ariQueryParams = []
} }
let rspBS = NC.responseBody rsp let rspBS = NC.responseBody rsp
@ -405,35 +458,44 @@ serviceStatus = do
serviceSendAction :: ServiceAction -> Minio () serviceSendAction :: ServiceAction -> Minio ()
serviceSendAction action = do serviceSendAction action = do
let payload = PayloadBS $ LBS.toStrict $ A.encode action let payload = PayloadBS $ LBS.toStrict $ A.encode action
void $ executeAdminRequest AdminReqInfo { ariMethod = HT.methodPost void $
, ariPayload = payload executeAdminRequest
, ariPayloadHash = Nothing AdminReqInfo
, ariPath = "v1/service" { ariMethod = HT.methodPost,
, ariHeaders = [] ariPayload = payload,
, ariQueryParams = [] ariPayloadHash = Nothing,
ariPath = "v1/service",
ariHeaders = [],
ariQueryParams = []
} }
-- | Get the current config file from server. -- | Get the current config file from server.
getConfig :: Minio ByteString getConfig :: Minio ByteString
getConfig = do getConfig = do
rsp <- executeAdminRequest AdminReqInfo { ariMethod = HT.methodGet rsp <-
, ariPayload = PayloadBS B.empty executeAdminRequest
, ariPayloadHash = Nothing AdminReqInfo
, ariPath = "v1/config" { ariMethod = HT.methodGet,
, ariHeaders = [] ariPayload = PayloadBS B.empty,
, ariQueryParams = [] ariPayloadHash = Nothing,
ariPath = "v1/config",
ariHeaders = [],
ariQueryParams = []
} }
return $ LBS.toStrict $ NC.responseBody rsp return $ LBS.toStrict $ NC.responseBody rsp
-- | Set a new config to the server. -- | Set a new config to the server.
setConfig :: ByteString -> Minio SetConfigResult setConfig :: ByteString -> Minio SetConfigResult
setConfig config = do setConfig config = do
rsp <- executeAdminRequest AdminReqInfo { ariMethod = HT.methodPut rsp <-
, ariPayload = PayloadBS config executeAdminRequest
, ariPayloadHash = Nothing AdminReqInfo
, ariPath = "v1/config" { ariMethod = HT.methodPut,
, ariHeaders = [] ariPayload = PayloadBS config,
, ariQueryParams = [] ariPayloadHash = Nothing,
ariPath = "v1/config",
ariHeaders = [],
ariQueryParams = []
} }
let rspBS = NC.responseBody rsp let rspBS = NC.responseBody rsp
@ -448,12 +510,15 @@ getHealStatus :: Maybe Bucket -> Maybe Text -> Text -> Minio HealStatus
getHealStatus bucket prefix token = do getHealStatus bucket prefix token = do
when (isNothing bucket && isJust prefix) $ throwIO MErrVInvalidHealPath when (isNothing bucket && isJust prefix) $ throwIO MErrVInvalidHealPath
let qparams = HT.queryTextToQuery [("clientToken", Just token)] let qparams = HT.queryTextToQuery [("clientToken", Just token)]
rsp <- executeAdminRequest AdminReqInfo { ariMethod = HT.methodPost rsp <-
, ariPayload = PayloadBS B.empty executeAdminRequest
, ariPayloadHash = Nothing AdminReqInfo
, ariPath = healPath bucket prefix { ariMethod = HT.methodPost,
, ariHeaders = [] ariPayload = PayloadBS B.empty,
, ariQueryParams = qparams ariPayloadHash = Nothing,
ariPath = healPath bucket prefix,
ariHeaders = [],
ariQueryParams = qparams
} }
let rspBS = NC.responseBody rsp let rspBS = NC.responseBody rsp
case eitherDecode rspBS of case eitherDecode rspBS of
@ -464,15 +529,21 @@ doHeal :: Maybe Bucket -> Maybe Text -> HealOpts -> Bool -> Minio HealStartResp
doHeal bucket prefix opts forceStart = do doHeal bucket prefix opts forceStart = do
when (isNothing bucket && isJust prefix) $ throwIO MErrVInvalidHealPath when (isNothing bucket && isJust prefix) $ throwIO MErrVInvalidHealPath
let payload = PayloadBS $ LBS.toStrict $ A.encode opts let payload = PayloadBS $ LBS.toStrict $ A.encode opts
let qparams = bool [] (HT.queryTextToQuery [("forceStart", Just "true")]) let qparams =
bool
[]
(HT.queryTextToQuery [("forceStart", Just "true")])
forceStart forceStart
rsp <- executeAdminRequest AdminReqInfo { ariMethod = HT.methodPost rsp <-
, ariPayload = payload executeAdminRequest
, ariPayloadHash = Nothing AdminReqInfo
, ariPath = healPath bucket prefix { ariMethod = HT.methodPost,
, ariHeaders = [] ariPayload = payload,
, ariQueryParams = qparams ariPayloadHash = Nothing,
ariPath = healPath bucket prefix,
ariHeaders = [],
ariQueryParams = qparams
} }
let rspBS = NC.responseBody rsp let rspBS = NC.responseBody rsp
@ -500,12 +571,15 @@ forceStartHeal bucket prefix opts = doHeal bucket prefix opts True
-- properties, storage information, network statistics, etc. -- properties, storage information, network statistics, etc.
getServerInfo :: Minio [ServerInfo] getServerInfo :: Minio [ServerInfo]
getServerInfo = do getServerInfo = do
rsp <- executeAdminRequest AdminReqInfo { ariMethod = HT.methodGet rsp <-
, ariPayload = PayloadBS B.empty executeAdminRequest
, ariPayloadHash = Nothing AdminReqInfo
, ariPath = "v1/info" { ariMethod = HT.methodGet,
, ariHeaders = [] ariPayload = PayloadBS B.empty,
, ariQueryParams = [] ariPayloadHash = Nothing,
ariPath = "v1/info",
ariHeaders = [],
ariQueryParams = []
} }
let rspBS = NC.responseBody rsp let rspBS = NC.responseBody rsp
case eitherDecode rspBS of case eitherDecode rspBS of
@ -521,39 +595,51 @@ executeAdminRequest ari = do
buildAdminRequest :: AdminReqInfo -> Minio NC.Request buildAdminRequest :: AdminReqInfo -> Minio NC.Request
buildAdminRequest areq = do buildAdminRequest areq = do
ci <- asks mcConnInfo ci <- asks mcConnInfo
sha256Hash <- if | connectIsSecure ci -> sha256Hash <-
if
| connectIsSecure ci ->
-- if secure connection -- if secure connection
return "UNSIGNED-PAYLOAD" return "UNSIGNED-PAYLOAD"
-- otherwise compute sha256 -- otherwise compute sha256
| otherwise -> getPayloadSHA256Hash (ariPayload areq) | otherwise -> getPayloadSHA256Hash (ariPayload areq)
timeStamp <- liftIO getCurrentTime timeStamp <- liftIO getCurrentTime
let hostHeader = (hHost, getHostAddr ci) let hostHeader = (hHost, getHostAddr ci)
newAreq = areq { ariPayloadHash = Just sha256Hash newAreq =
, ariHeaders = hostHeader areq
{ ariPayloadHash = Just sha256Hash,
ariHeaders =
hostHeader
: sha256Header sha256Hash : sha256Header sha256Hash
: ariHeaders areq : ariHeaders areq
} }
signReq = toRequest ci newAreq signReq = toRequest ci newAreq
sp = SignParams (connectAccessKey ci) (connectSecretKey ci) sp =
timeStamp Nothing Nothing (ariPayloadHash newAreq) SignParams
(connectAccessKey ci)
(connectSecretKey ci)
timeStamp
Nothing
Nothing
(ariPayloadHash newAreq)
signHeaders = signV4 sp signReq signHeaders = signV4 sp signReq
-- Update signReq with Authorization header containing v4 signature -- Update signReq with Authorization header containing v4 signature
return signReq { return
NC.requestHeaders = ariHeaders newAreq ++ mkHeaderFromPairs signHeaders signReq
{ NC.requestHeaders = ariHeaders newAreq ++ mkHeaderFromPairs signHeaders
} }
where where
toRequest :: ConnectInfo -> AdminReqInfo -> NC.Request toRequest :: ConnectInfo -> AdminReqInfo -> NC.Request
toRequest ci aReq = NC.defaultRequest toRequest ci aReq =
{ NC.method = ariMethod aReq NC.defaultRequest
, NC.secure = connectIsSecure ci { NC.method = ariMethod aReq,
, NC.host = encodeUtf8 $ connectHost ci NC.secure = connectIsSecure ci,
, NC.port = connectPort ci NC.host = encodeUtf8 $ connectHost ci,
, NC.path = B.intercalate "/" [adminPath, ariPath aReq] NC.port = connectPort ci,
, NC.requestHeaders = ariHeaders aReq NC.path = B.intercalate "/" [adminPath, ariPath aReq],
, NC.queryString = HT.renderQuery False $ ariQueryParams aReq NC.requestHeaders = ariHeaders aReq,
, NC.requestBody = getRequestBody (ariPayload aReq) NC.queryString = HT.renderQuery False $ ariQueryParams aReq,
NC.requestBody = getRequestBody (ariPayload aReq)
} }

View File

@ -17,18 +17,18 @@
module Network.Minio.CopyObject where module Network.Minio.CopyObject where
import qualified Data.List as List import qualified Data.List as List
import Lib.Prelude import Lib.Prelude
import Network.Minio.Data import Network.Minio.Data
import Network.Minio.Errors import Network.Minio.Errors
import Network.Minio.S3API import Network.Minio.S3API
import Network.Minio.Utils import Network.Minio.Utils
-- | Copy an object using single or multipart copy strategy. -- | Copy an object using single or multipart copy strategy.
copyObjectInternal :: Bucket -> Object -> SourceInfo copyObjectInternal ::
-> Minio ETag Bucket ->
Object ->
SourceInfo ->
Minio ETag
copyObjectInternal b' o srcInfo = do copyObjectInternal b' o srcInfo = do
let sBucket = srcBucket srcInfo let sBucket = srcBucket srcInfo
sObject = srcObject srcInfo sObject = srcObject srcInfo
@ -43,27 +43,35 @@ copyObjectInternal b' o srcInfo = do
startOffset = fst range startOffset = fst range
endOffset = snd range endOffset = snd range
when (isJust rangeMay && when
or [startOffset < 0, endOffset < startOffset, ( isJust rangeMay
endOffset >= fromIntegral srcSize]) $ && or
throwIO $ MErrVInvalidSrcObjByteRange range [ startOffset < 0,
endOffset < startOffset,
endOffset >= fromIntegral srcSize
]
)
$ throwIO
$ MErrVInvalidSrcObjByteRange range
-- 1. If sz > 64MiB (minPartSize) use multipart copy, OR -- 1. If sz > 64MiB (minPartSize) use multipart copy, OR
-- 2. If startOffset /= 0 use multipart copy -- 2. If startOffset /= 0 use multipart copy
let destSize = (\(a, b) -> b - a + 1 ) $ let destSize =
(\(a, b) -> b - a + 1) $
maybe (0, srcSize - 1) identity rangeMay maybe (0, srcSize - 1) identity rangeMay
if destSize > minPartSize || (endOffset - startOffset + 1 /= srcSize) if destSize > minPartSize || (endOffset - startOffset + 1 /= srcSize)
then multiPartCopyObject b' o srcInfo srcSize then multiPartCopyObject b' o srcInfo srcSize
else fst <$> copyObjectSingle b' o srcInfo {srcRange = Nothing} []
else fst <$> copyObjectSingle b' o srcInfo{srcRange = Nothing} []
-- | Given the input byte range of the source object, compute the -- | Given the input byte range of the source object, compute the
-- splits for a multipart copy object procedure. Minimum part size -- splits for a multipart copy object procedure. Minimum part size
-- used is minPartSize. -- used is minPartSize.
selectCopyRanges :: (Int64, Int64) -> [(PartNumber, (Int64, Int64))] selectCopyRanges :: (Int64, Int64) -> [(PartNumber, (Int64, Int64))]
selectCopyRanges (st, end) = zip pns $ selectCopyRanges (st, end) =
map (\(x, y) -> (st + x, st + x + y - 1)) $ zip startOffsets partSizes zip pns
$ map (\(x, y) -> (st + x, st + x + y - 1))
$ zip startOffsets partSizes
where where
size = end - st + 1 size = end - st + 1
(pns, startOffsets, partSizes) = List.unzip3 $ selectPartSizes size (pns, startOffsets, partSizes) = List.unzip3 $ selectPartSizes size
@ -71,19 +79,27 @@ selectCopyRanges (st, end) = zip pns $
-- | Perform a multipart copy object action. Since we cannot verify -- | Perform a multipart copy object action. Since we cannot verify
-- existing parts based on the source object, there is no resuming -- existing parts based on the source object, there is no resuming
-- copy action support. -- copy action support.
multiPartCopyObject :: Bucket -> Object -> SourceInfo -> Int64 multiPartCopyObject ::
-> Minio ETag Bucket ->
Object ->
SourceInfo ->
Int64 ->
Minio ETag
multiPartCopyObject b o cps srcSize = do multiPartCopyObject b o cps srcSize = do
uid <- newMultipartUpload b o [] uid <- newMultipartUpload b o []
let byteRange = maybe (0, fromIntegral $ srcSize - 1) identity $ srcRange cps let byteRange = maybe (0, fromIntegral $ srcSize - 1) identity $ srcRange cps
partRanges = selectCopyRanges byteRange partRanges = selectCopyRanges byteRange
partSources = map (\(x, (start, end)) -> (x, cps {srcRange = Just (start, end) })) partSources =
map
(\(x, (start, end)) -> (x, cps {srcRange = Just (start, end)}))
partRanges partRanges
dstInfo = defaultDestinationInfo { dstBucket = b, dstObject = o} dstInfo = defaultDestinationInfo {dstBucket = b, dstObject = o}
copiedParts <- limitedMapConcurrently 10 copiedParts <-
(\(pn, cps') -> do limitedMapConcurrently
10
( \(pn, cps') -> do
(etag, _) <- copyObjectPart dstInfo cps' uid pn [] (etag, _) <- copyObjectPart dstInfo cps' uid pn []
return (pn, etag) return (pn, etag)
) )

File diff suppressed because it is too large Load Diff

View File

@ -13,23 +13,22 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE FlexibleInstances #-}
module Network.Minio.Data.ByteString module Network.Minio.Data.ByteString
( ( stripBS,
stripBS UriEncodable (..),
, UriEncodable(..) )
) where where
import qualified Data.ByteString as B import qualified Data.ByteString as B
import qualified Data.ByteString.Builder as BB import qualified Data.ByteString.Builder as BB
import qualified Data.ByteString.Char8 as BC8 import qualified Data.ByteString.Char8 as BC8
import qualified Data.ByteString.Lazy as LB import qualified Data.ByteString.Lazy as LB
import Data.Char (isSpace, toUpper, isAsciiUpper, isAsciiLower, isDigit) import Data.Char (isAsciiLower, isAsciiUpper)
import qualified Data.Text as T import qualified Data.Text as T
import Numeric (showHex)
import Lib.Prelude import Lib.Prelude
import Numeric (showHex)
stripBS :: ByteString -> ByteString stripBS :: ByteString -> ByteString
stripBS = BC8.dropWhile isSpace . fst . BC8.spanEnd isSpace stripBS = BC8.dropWhile isSpace . fst . BC8.spanEnd isSpace
@ -64,11 +63,12 @@ uriEncodeChar ch _
|| (ch == '_') || (ch == '_')
|| (ch == '-') || (ch == '-')
|| (ch == '.') || (ch == '.')
|| (ch == '~') = BB.char7 ch || (ch == '~') =
BB.char7 ch
| otherwise = mconcat $ map f $ B.unpack $ encodeUtf8 $ T.singleton ch | otherwise = mconcat $ map f $ B.unpack $ encodeUtf8 $ T.singleton ch
where where
f :: Word8 -> BB.Builder f :: Word8 -> BB.Builder
f n = BB.char7 '%' <> BB.string7 hexStr f n = BB.char7 '%' <> BB.string7 hexStr
where where
hexStr = map toUpper $ showHex q $ showHex r "" hexStr = map toUpper $ showHex q $ showHex r ""
(q, r) = divMod (fromIntegral n) (16::Word8) (q, r) = divMod (fromIntegral n) (16 :: Word8)

View File

@ -15,30 +15,30 @@
-- --
module Network.Minio.Data.Crypto module Network.Minio.Data.Crypto
( ( hashSHA256,
hashSHA256 hashSHA256FromSource,
, hashSHA256FromSource hashMD5,
hashMD5ToBase64,
hashMD5FromSource,
hmacSHA256,
hmacSHA256RawBS,
digestToBS,
digestToBase16,
encodeToBase64,
)
where
, hashMD5 import Crypto.Hash
, hashMD5ToBase64 ( Digest,
, hashMD5FromSource MD5 (..),
SHA256 (..),
, hmacSHA256 hashWith,
, hmacSHA256RawBS )
, digestToBS
, digestToBase16
, encodeToBase64
) where
import Crypto.Hash (Digest, MD5 (..), SHA256 (..),
hashWith)
import Crypto.Hash.Conduit (sinkHash) import Crypto.Hash.Conduit (sinkHash)
import Crypto.MAC.HMAC (HMAC, hmac) import Crypto.MAC.HMAC (HMAC, hmac)
import Data.ByteArray (ByteArrayAccess, convert) import Data.ByteArray (ByteArrayAccess, convert)
import Data.ByteArray.Encoding (Base (Base16, Base64), convertToBase) import Data.ByteArray.Encoding (Base (Base16, Base64), convertToBase)
import qualified Data.Conduit as C import qualified Data.Conduit as C
import Lib.Prelude import Lib.Prelude
hashSHA256 :: ByteString -> ByteString hashSHA256 :: ByteString -> ByteString

View File

@ -15,19 +15,17 @@
-- --
module Network.Minio.Data.Time module Network.Minio.Data.Time
( ( awsTimeFormat,
awsTimeFormat awsTimeFormatBS,
, awsTimeFormatBS awsDateFormat,
, awsDateFormat awsDateFormatBS,
, awsDateFormatBS awsParseTime,
, awsParseTime iso8601TimeFormat,
, iso8601TimeFormat )
) where where
import Data.ByteString.Char8 (pack) import Data.ByteString.Char8 (pack)
import qualified Data.Time as Time import qualified Data.Time as Time
import Lib.Prelude import Lib.Prelude
awsTimeFormat :: UTCTime -> [Char] awsTimeFormat :: UTCTime -> [Char]

View File

@ -17,16 +17,16 @@
module Network.Minio.Errors where module Network.Minio.Errors where
import Control.Exception import Control.Exception
import qualified Network.HTTP.Conduit as NC
import Lib.Prelude import Lib.Prelude
import qualified Network.HTTP.Conduit as NC
--------------------------------- ---------------------------------
-- Errors -- Errors
--------------------------------- ---------------------------------
-- | Various validation errors -- | Various validation errors
data MErrV = MErrVSinglePUTSizeExceeded Int64 data MErrV
= MErrVSinglePUTSizeExceeded Int64
| MErrVPutSizeExceeded Int64 | MErrVPutSizeExceeded Int64
| MErrVETagHeaderNotFound | MErrVETagHeaderNotFound
| MErrVInvalidObjectInfoResponse | MErrVInvalidObjectInfoResponse
@ -49,7 +49,8 @@ data MErrV = MErrVSinglePUTSizeExceeded Int64
instance Exception MErrV instance Exception MErrV
-- | Errors returned by S3 compatible service -- | Errors returned by S3 compatible service
data ServiceErr = BucketAlreadyExists data ServiceErr
= BucketAlreadyExists
| BucketAlreadyOwnedByYou | BucketAlreadyOwnedByYou
| NoSuchBucket | NoSuchBucket
| InvalidBucketName | InvalidBucketName
@ -68,9 +69,9 @@ toServiceErr "BucketAlreadyOwnedByYou" _ = BucketAlreadyOwnedByYou
toServiceErr "BucketAlreadyExists" _ = BucketAlreadyExists toServiceErr "BucketAlreadyExists" _ = BucketAlreadyExists
toServiceErr code message = ServiceErr code message toServiceErr code message = ServiceErr code message
-- | Errors thrown by the library -- | Errors thrown by the library
data MinioErr = MErrHTTP NC.HttpException data MinioErr
= MErrHTTP NC.HttpException
| MErrIO IOException | MErrIO IOException
| MErrService ServiceErr | MErrService ServiceErr
| MErrValidation MErrV | MErrValidation MErrV

View File

@ -15,23 +15,30 @@
-- --
module Network.Minio.JsonParser module Network.Minio.JsonParser
( ( parseErrResponseJSON,
parseErrResponseJSON )
) where where
import Data.Aeson (FromJSON, eitherDecode, parseJSON, import Data.Aeson
withObject, (.:)) ( (.:),
FromJSON,
eitherDecode,
parseJSON,
withObject,
)
import qualified Data.Text as T import qualified Data.Text as T
import Lib.Prelude import Lib.Prelude
import Network.Minio.Errors import Network.Minio.Errors
data AdminErrJSON = AdminErrJSON { aeCode :: Text data AdminErrJSON = AdminErrJSON
, aeMessage :: Text { aeCode :: Text,
} deriving (Eq, Show) aeMessage :: Text
}
deriving (Eq, Show)
instance FromJSON AdminErrJSON where instance FromJSON AdminErrJSON where
parseJSON = withObject "AdminErrJSON" $ \v -> AdminErrJSON parseJSON = withObject "AdminErrJSON" $ \v ->
AdminErrJSON
<$> v .: "Code" <$> v .: "Code"
<*> v .: "Message" <*> v .: "Message"

View File

@ -19,15 +19,14 @@ module Network.Minio.ListOps where
import qualified Data.Conduit as C import qualified Data.Conduit as C
import qualified Data.Conduit.Combinators as CC import qualified Data.Conduit.Combinators as CC
import qualified Data.Conduit.List as CL import qualified Data.Conduit.List as CL
import Lib.Prelude import Lib.Prelude
import Network.Minio.Data import Network.Minio.Data
import Network.Minio.S3API import Network.Minio.S3API
-- | Represents a list output item - either an object or an object -- | Represents a list output item - either an object or an object
-- prefix (i.e. a directory). -- prefix (i.e. a directory).
data ListItem = ListItemObject ObjectInfo data ListItem
= ListItemObject ObjectInfo
| ListItemPrefix Text | ListItemPrefix Text
deriving (Show, Eq) deriving (Show, Eq)
@ -48,72 +47,98 @@ listObjects bucket prefix recurse = loop Nothing
where where
loop :: Maybe Text -> C.ConduitM () ListItem Minio () loop :: Maybe Text -> C.ConduitM () ListItem Minio ()
loop nextToken = do loop nextToken = do
let let delimiter = bool (Just "/") Nothing recurse
delimiter = bool (Just "/") Nothing recurse
res <- lift $ listObjects' bucket prefix nextToken delimiter Nothing res <- lift $ listObjects' bucket prefix nextToken delimiter Nothing
CL.sourceList $ map ListItemObject $ lorObjects res CL.sourceList $ map ListItemObject $ lorObjects res
unless recurse $ unless recurse
CL.sourceList $ map ListItemPrefix $ lorCPrefixes res $ CL.sourceList
$ map ListItemPrefix
$ lorCPrefixes res
when (lorHasMore res) $ when (lorHasMore res) $
loop (lorNextToken res) loop (lorNextToken res)
-- | Lists objects - similar to @listObjects@, however uses the older -- | Lists objects - similar to @listObjects@, however uses the older
-- V1 AWS S3 API. Prefer @listObjects@ to this. -- V1 AWS S3 API. Prefer @listObjects@ to this.
listObjectsV1 :: Bucket -> Maybe Text -> Bool listObjectsV1 ::
-> C.ConduitM () ListItem Minio () Bucket ->
Maybe Text ->
Bool ->
C.ConduitM () ListItem Minio ()
listObjectsV1 bucket prefix recurse = loop Nothing listObjectsV1 bucket prefix recurse = loop Nothing
where where
loop :: Maybe Text -> C.ConduitM () ListItem Minio () loop :: Maybe Text -> C.ConduitM () ListItem Minio ()
loop nextMarker = do loop nextMarker = do
let let delimiter = bool (Just "/") Nothing recurse
delimiter = bool (Just "/") Nothing recurse
res <- lift $ listObjectsV1' bucket prefix nextMarker delimiter Nothing res <- lift $ listObjectsV1' bucket prefix nextMarker delimiter Nothing
CL.sourceList $ map ListItemObject $ lorObjects' res CL.sourceList $ map ListItemObject $ lorObjects' res
unless recurse $ unless recurse
CL.sourceList $ map ListItemPrefix $ lorCPrefixes' res $ CL.sourceList
$ map ListItemPrefix
$ lorCPrefixes' res
when (lorHasMore' res) $ when (lorHasMore' res) $
loop (lorNextMarker res) loop (lorNextMarker res)
-- | List incomplete uploads in a bucket matching the given prefix. If -- | List incomplete uploads in a bucket matching the given prefix. If
-- recurse is set to True incomplete uploads for the given prefix are -- recurse is set to True incomplete uploads for the given prefix are
-- recursively listed. -- recursively listed.
listIncompleteUploads :: Bucket -> Maybe Text -> Bool listIncompleteUploads ::
-> C.ConduitM () UploadInfo Minio () Bucket ->
Maybe Text ->
Bool ->
C.ConduitM () UploadInfo Minio ()
listIncompleteUploads bucket prefix recurse = loop Nothing Nothing listIncompleteUploads bucket prefix recurse = loop Nothing Nothing
where where
loop :: Maybe Text -> Maybe Text -> C.ConduitM () UploadInfo Minio () loop :: Maybe Text -> Maybe Text -> C.ConduitM () UploadInfo Minio ()
loop nextKeyMarker nextUploadIdMarker = do loop nextKeyMarker nextUploadIdMarker = do
let let delimiter = bool (Just "/") Nothing recurse
delimiter = bool (Just "/") Nothing recurse
res <- lift $ listIncompleteUploads' bucket prefix delimiter res <-
nextKeyMarker nextUploadIdMarker Nothing lift $
listIncompleteUploads'
bucket
prefix
delimiter
nextKeyMarker
nextUploadIdMarker
Nothing
aggrSizes <- lift $ forM (lurUploads res) $ \(uKey, uId, _) -> do aggrSizes <- lift $ forM (lurUploads res) $ \(uKey, uId, _) -> do
partInfos <- C.runConduit $ listIncompleteParts bucket uKey uId partInfos <-
C.runConduit $
listIncompleteParts bucket uKey uId
C..| CC.sinkList C..| CC.sinkList
return $ foldl (\sizeSofar p -> opiSize p + sizeSofar) 0 partInfos return $ foldl (\sizeSofar p -> opiSize p + sizeSofar) 0 partInfos
CL.sourceList $ CL.sourceList
map (\((uKey, uId, uInitTime), size) -> $ map
( \((uKey, uId, uInitTime), size) ->
UploadInfo uKey uId uInitTime size UploadInfo uKey uId uInitTime size
) $ zip (lurUploads res) aggrSizes )
$ zip (lurUploads res) aggrSizes
when (lurHasMore res) $ when (lurHasMore res) $
loop (lurNextKey res) (lurNextUpload res) loop (lurNextKey res) (lurNextUpload res)
-- | List object parts of an ongoing multipart upload for given -- | List object parts of an ongoing multipart upload for given
-- bucket, object and uploadId. -- bucket, object and uploadId.
listIncompleteParts :: Bucket -> Object -> UploadId listIncompleteParts ::
-> C.ConduitM () ObjectPartInfo Minio () Bucket ->
Object ->
UploadId ->
C.ConduitM () ObjectPartInfo Minio ()
listIncompleteParts bucket object uploadId = loop Nothing listIncompleteParts bucket object uploadId = loop Nothing
where where
loop :: Maybe Text -> C.ConduitM () ObjectPartInfo Minio () loop :: Maybe Text -> C.ConduitM () ObjectPartInfo Minio ()
loop nextPartMarker = do loop nextPartMarker = do
res <- lift $ listIncompleteParts' bucket object uploadId Nothing res <-
lift $
listIncompleteParts'
bucket
object
uploadId
Nothing
nextPartMarker nextPartMarker
CL.sourceList $ lprParts res CL.sourceList $ lprParts res
when (lprHasMore res) $ when (lprHasMore res) $

View File

@ -15,26 +15,25 @@
-- --
module Network.Minio.PresignedOperations module Network.Minio.PresignedOperations
( UrlExpiry ( UrlExpiry,
, makePresignedUrl makePresignedUrl,
, presignedPutObjectUrl presignedPutObjectUrl,
, presignedGetObjectUrl presignedGetObjectUrl,
, presignedHeadObjectUrl presignedHeadObjectUrl,
PostPolicyCondition (..),
, PostPolicyCondition(..) ppCondBucket,
, ppCondBucket ppCondContentLengthRange,
, ppCondContentLengthRange ppCondContentType,
, ppCondContentType ppCondKey,
, ppCondKey ppCondKeyStartsWith,
, ppCondKeyStartsWith ppCondSuccessActionStatus,
, ppCondSuccessActionStatus PostPolicy (..),
PostPolicyError (..),
, PostPolicy(..) newPostPolicy,
, PostPolicyError(..) showPostPolicy,
, newPostPolicy presignedPostPolicy,
, showPostPolicy )
, presignedPostPolicy where
) where
import Data.Aeson ((.=)) import Data.Aeson ((.=))
import qualified Data.Aeson as Json import qualified Data.Aeson as Json
@ -42,12 +41,10 @@ import Data.ByteString.Builder (byteString, toLazyByteString)
import qualified Data.HashMap.Strict as H import qualified Data.HashMap.Strict as H
import qualified Data.Text as T import qualified Data.Text as T
import qualified Data.Time as Time import qualified Data.Time as Time
import Lib.Prelude
import qualified Network.HTTP.Conduit as NC import qualified Network.HTTP.Conduit as NC
import qualified Network.HTTP.Types as HT import qualified Network.HTTP.Types as HT
import Network.HTTP.Types.Header (hHost) import Network.HTTP.Types.Header (hHost)
import Lib.Prelude
import Network.Minio.Data import Network.Minio.Data
import Network.Minio.Data.Time import Network.Minio.Data.Time
import Network.Minio.Errors import Network.Minio.Errors
@ -61,39 +58,53 @@ import Network.Minio.Sign.V4
-- --
-- All extra query parameters or headers are signed, and therefore are -- All extra query parameters or headers are signed, and therefore are
-- required to be sent when the generated URL is actually used. -- required to be sent when the generated URL is actually used.
makePresignedUrl :: UrlExpiry -> HT.Method -> Maybe Bucket -> Maybe Object makePresignedUrl ::
-> Maybe Region -> HT.Query -> HT.RequestHeaders UrlExpiry ->
-> Minio ByteString HT.Method ->
Maybe Bucket ->
Maybe Object ->
Maybe Region ->
HT.Query ->
HT.RequestHeaders ->
Minio ByteString
makePresignedUrl expiry method bucket object region extraQuery extraHeaders = do makePresignedUrl expiry method bucket object region extraQuery extraHeaders = do
when (expiry > 7*24*3600 || expiry < 0) $ when (expiry > 7 * 24 * 3600 || expiry < 0)
throwIO $ MErrVInvalidUrlExpiry expiry $ throwIO
$ MErrVInvalidUrlExpiry expiry
ci <- asks mcConnInfo ci <- asks mcConnInfo
let let hostHeader = (hHost, getHostAddr ci)
hostHeader = (hHost, getHostAddr ci) req =
req = NC.defaultRequest { NC.defaultRequest
NC.method = method { NC.method = method,
, NC.secure = connectIsSecure ci NC.secure = connectIsSecure ci,
, NC.host = encodeUtf8 $ connectHost ci NC.host = encodeUtf8 $ connectHost ci,
, NC.port = connectPort ci NC.port = connectPort ci,
, NC.path = getS3Path bucket object NC.path = getS3Path bucket object,
, NC.requestHeaders = hostHeader : extraHeaders NC.requestHeaders = hostHeader : extraHeaders,
, NC.queryString = HT.renderQuery True extraQuery NC.queryString = HT.renderQuery True extraQuery
} }
ts <- liftIO Time.getCurrentTime ts <- liftIO Time.getCurrentTime
let sp = SignParams (connectAccessKey ci) (connectSecretKey ci) let sp =
ts region (Just expiry) Nothing SignParams
(connectAccessKey ci)
(connectSecretKey ci)
ts
region
(Just expiry)
Nothing
signPairs = signV4 sp req signPairs = signV4 sp req
qpToAdd = (fmap . fmap) Just signPairs qpToAdd = (fmap . fmap) Just signPairs
queryStr = HT.renderQueryBuilder True queryStr =
HT.renderQueryBuilder
True
((HT.parseQuery $ NC.queryString req) ++ qpToAdd) ((HT.parseQuery $ NC.queryString req) ++ qpToAdd)
scheme = byteString $ bool "http://" "https://" $ connectIsSecure ci scheme = byteString $ bool "http://" "https://" $ connectIsSecure ci
return $ toS $ toLazyByteString $ scheme return $ toStrictBS $ toLazyByteString $
scheme
<> byteString (getHostAddr ci) <> byteString (getHostAddr ci)
<> byteString (getS3Path bucket object) <> byteString (getS3Path bucket object)
<> queryStr <> queryStr
@ -105,12 +116,22 @@ makePresignedUrl expiry method bucket object region extraQuery extraHeaders = do
-- --
-- For a list of possible headers to pass, please refer to the PUT -- For a list of possible headers to pass, please refer to the PUT
-- object REST API AWS S3 documentation. -- object REST API AWS S3 documentation.
presignedPutObjectUrl :: Bucket -> Object -> UrlExpiry -> HT.RequestHeaders presignedPutObjectUrl ::
-> Minio ByteString Bucket ->
Object ->
UrlExpiry ->
HT.RequestHeaders ->
Minio ByteString
presignedPutObjectUrl bucket object expirySeconds extraHeaders = do presignedPutObjectUrl bucket object expirySeconds extraHeaders = do
region <- asks (Just . connectRegion . mcConnInfo) region <- asks (Just . connectRegion . mcConnInfo)
makePresignedUrl expirySeconds HT.methodPut makePresignedUrl
(Just bucket) (Just object) region [] extraHeaders expirySeconds
HT.methodPut
(Just bucket)
(Just object)
region
[]
extraHeaders
-- | Generate a URL with authentication signature to GET (download) an -- | Generate a URL with authentication signature to GET (download) an
-- object. All extra query parameters and headers passed here will be -- object. All extra query parameters and headers passed here will be
@ -121,12 +142,23 @@ presignedPutObjectUrl bucket object expirySeconds extraHeaders = do
-- --
-- For a list of possible request parameters and headers, please refer -- For a list of possible request parameters and headers, please refer
-- to the GET object REST API AWS S3 documentation. -- to the GET object REST API AWS S3 documentation.
presignedGetObjectUrl :: Bucket -> Object -> UrlExpiry -> HT.Query presignedGetObjectUrl ::
-> HT.RequestHeaders -> Minio ByteString Bucket ->
Object ->
UrlExpiry ->
HT.Query ->
HT.RequestHeaders ->
Minio ByteString
presignedGetObjectUrl bucket object expirySeconds extraQuery extraHeaders = do presignedGetObjectUrl bucket object expirySeconds extraQuery extraHeaders = do
region <- asks (Just . connectRegion . mcConnInfo) region <- asks (Just . connectRegion . mcConnInfo)
makePresignedUrl expirySeconds HT.methodGet makePresignedUrl
(Just bucket) (Just object) region extraQuery extraHeaders expirySeconds
HT.methodGet
(Just bucket)
(Just object)
region
extraQuery
extraHeaders
-- | Generate a URL with authentication signature to make a HEAD -- | Generate a URL with authentication signature to make a HEAD
-- request on an object. This is used to fetch metadata about an -- request on an object. This is used to fetch metadata about an
@ -135,15 +167,26 @@ presignedGetObjectUrl bucket object expirySeconds extraQuery extraHeaders = do
-- --
-- For a list of possible headers to pass, please refer to the HEAD -- For a list of possible headers to pass, please refer to the HEAD
-- object REST API AWS S3 documentation. -- object REST API AWS S3 documentation.
presignedHeadObjectUrl :: Bucket -> Object -> UrlExpiry presignedHeadObjectUrl ::
-> HT.RequestHeaders -> Minio ByteString Bucket ->
Object ->
UrlExpiry ->
HT.RequestHeaders ->
Minio ByteString
presignedHeadObjectUrl bucket object expirySeconds extraHeaders = do presignedHeadObjectUrl bucket object expirySeconds extraHeaders = do
region <- asks (Just . connectRegion . mcConnInfo) region <- asks (Just . connectRegion . mcConnInfo)
makePresignedUrl expirySeconds HT.methodHead makePresignedUrl
(Just bucket) (Just object) region [] extraHeaders expirySeconds
HT.methodHead
(Just bucket)
(Just object)
region
[]
extraHeaders
-- | Represents individual conditions in a Post Policy document. -- | Represents individual conditions in a Post Policy document.
data PostPolicyCondition = PPCStartsWith Text Text data PostPolicyCondition
= PPCStartsWith Text Text
| PPCEquals Text Text | PPCEquals Text Text
| PPCRange Text Int64 Int64 | PPCRange Text Int64 Int64
deriving (Show, Eq) deriving (Show, Eq)
@ -160,21 +203,24 @@ instance Json.ToJSON PostPolicyCondition where
Json.foldable [Json.toJSON k, Json.toJSON minVal, Json.toJSON maxVal] Json.foldable [Json.toJSON k, Json.toJSON minVal, Json.toJSON maxVal]
-- | A PostPolicy is required to perform uploads via browser forms. -- | A PostPolicy is required to perform uploads via browser forms.
data PostPolicy = PostPolicy { data PostPolicy = PostPolicy
expiration :: UTCTime { expiration :: UTCTime,
, conditions :: [PostPolicyCondition] conditions :: [PostPolicyCondition]
} deriving (Show, Eq) }
deriving (Show, Eq)
instance Json.ToJSON PostPolicy where instance Json.ToJSON PostPolicy where
toJSON (PostPolicy e c) = toJSON (PostPolicy e c) =
Json.object $ [ "expiration" .= iso8601TimeFormat e Json.object $
, "conditions" .= c [ "expiration" .= iso8601TimeFormat e,
"conditions" .= c
] ]
toEncoding (PostPolicy e c) = toEncoding (PostPolicy e c) =
Json.pairs ("expiration" .= iso8601TimeFormat e <> "conditions" .= c) Json.pairs ("expiration" .= iso8601TimeFormat e <> "conditions" .= c)
-- | Possible validation errors when creating a PostPolicy. -- | Possible validation errors when creating a PostPolicy.
data PostPolicyError = PPEKeyNotSpecified data PostPolicyError
= PPEKeyNotSpecified
| PPEBucketNotSpecified | PPEBucketNotSpecified
| PPEConditionKeyEmpty | PPEConditionKeyEmpty
| PPERangeInvalid | PPERangeInvalid
@ -186,8 +232,10 @@ ppCondBucket = PPCEquals "bucket"
-- | Set the content length range constraint with minimum and maximum -- | Set the content length range constraint with minimum and maximum
-- byte count values. -- byte count values.
ppCondContentLengthRange :: Int64 -> Int64 ppCondContentLengthRange ::
-> PostPolicyCondition Int64 ->
Int64 ->
PostPolicyCondition
ppCondContentLengthRange = PPCRange "content-length-range" ppCondContentLengthRange = PPCRange "content-length-range"
-- | Set the content-type header for the upload. -- | Set the content-type header for the upload.
@ -210,83 +258,91 @@ ppCondSuccessActionStatus n =
-- | This function creates a PostPolicy after validating its -- | This function creates a PostPolicy after validating its
-- arguments. -- arguments.
newPostPolicy :: UTCTime -> [PostPolicyCondition] newPostPolicy ::
-> Either PostPolicyError PostPolicy UTCTime ->
[PostPolicyCondition] ->
Either PostPolicyError PostPolicy
newPostPolicy expirationTime conds newPostPolicy expirationTime conds
-- object name condition must be present -- object name condition must be present
| not $ any (keyEquals "key") conds = | not $ any (keyEquals "key") conds =
Left PPEKeyNotSpecified Left PPEKeyNotSpecified
-- bucket name condition must be present -- bucket name condition must be present
| not $ any (keyEquals "bucket") conds = | not $ any (keyEquals "bucket") conds =
Left PPEBucketNotSpecified Left PPEBucketNotSpecified
-- a condition with an empty key is invalid -- a condition with an empty key is invalid
| any (keyEquals "") conds || any isEmptyRangeKey conds = | any (keyEquals "") conds || any isEmptyRangeKey conds =
Left PPEConditionKeyEmpty Left PPEConditionKeyEmpty
-- invalid range check -- invalid range check
| any isInvalidRange conds = | any isInvalidRange conds =
Left PPERangeInvalid Left PPERangeInvalid
-- all good! -- all good!
| otherwise = | otherwise =
return $ PostPolicy expirationTime conds return $ PostPolicy expirationTime conds
where where
keyEquals k' (PPCStartsWith k _) = k == k' keyEquals k' (PPCStartsWith k _) = k == k'
keyEquals k' (PPCEquals k _) = k == k' keyEquals k' (PPCEquals k _) = k == k'
keyEquals _ _ = False keyEquals _ _ = False
isEmptyRangeKey (PPCRange k _ _) = k == "" isEmptyRangeKey (PPCRange k _ _) = k == ""
isEmptyRangeKey _ = False isEmptyRangeKey _ = False
isInvalidRange (PPCRange _ mi ma) = mi < 0 || mi > ma isInvalidRange (PPCRange _ mi ma) = mi < 0 || mi > ma
isInvalidRange _ = False isInvalidRange _ = False
-- | Convert Post Policy to a string (e.g. for printing). -- | Convert Post Policy to a string (e.g. for printing).
showPostPolicy :: PostPolicy -> ByteString showPostPolicy :: PostPolicy -> ByteString
showPostPolicy = toS . Json.encode showPostPolicy = toStrictBS . Json.encode
-- | Generate a presigned URL and POST policy to upload files via a -- | Generate a presigned URL and POST policy to upload files via a
-- browser. On success, this function returns a URL and POST -- browser. On success, this function returns a URL and POST
-- form-data. -- form-data.
presignedPostPolicy :: PostPolicy presignedPostPolicy ::
-> Minio (ByteString, H.HashMap Text ByteString) PostPolicy ->
Minio (ByteString, H.HashMap Text ByteString)
presignedPostPolicy p = do presignedPostPolicy p = do
ci <- asks mcConnInfo ci <- asks mcConnInfo
signTime <- liftIO $ Time.getCurrentTime signTime <- liftIO $ Time.getCurrentTime
let let extraConditions =
extraConditions = [ PPCEquals "x-amz-date" (toS $ awsTimeFormat signTime),
[ PPCEquals "x-amz-date" (toS $ awsTimeFormat signTime) PPCEquals "x-amz-algorithm" "AWS4-HMAC-SHA256",
, PPCEquals "x-amz-algorithm" "AWS4-HMAC-SHA256" PPCEquals
, PPCEquals "x-amz-credential" "x-amz-credential"
(T.intercalate "/" [connectAccessKey ci, ( T.intercalate
decodeUtf8 $ mkScope signTime region]) "/"
[ connectAccessKey ci,
decodeUtf8 $ mkScope signTime region
] ]
ppWithCreds = p { )
conditions = conditions p ++ extraConditions ]
ppWithCreds =
p
{ conditions = conditions p ++ extraConditions
} }
sp = SignParams (connectAccessKey ci) (connectSecretKey ci) sp =
signTime (Just $ connectRegion ci) Nothing Nothing SignParams
(connectAccessKey ci)
(connectSecretKey ci)
signTime
(Just $ connectRegion ci)
Nothing
Nothing
signData = signV4PostPolicy (showPostPolicy ppWithCreds) sp signData = signV4PostPolicy (showPostPolicy ppWithCreds) sp
-- compute form-data -- compute form-data
mkPair (PPCStartsWith k v) = Just (k, v) mkPair (PPCStartsWith k v) = Just (k, v)
mkPair (PPCEquals k v) = Just (k, v) mkPair (PPCEquals k v) = Just (k, v)
mkPair _ = Nothing mkPair _ = Nothing
formFromPolicy = H.map toS $ H.fromList $ catMaybes $ formFromPolicy =
H.map toUtf8 $ H.fromList $ catMaybes $
mkPair <$> conditions ppWithCreds mkPair <$> conditions ppWithCreds
formData = formFromPolicy `H.union` signData formData = formFromPolicy `H.union` signData
-- compute POST upload URL -- compute POST upload URL
bucket = H.lookupDefault "" "bucket" formData bucket = H.lookupDefault "" "bucket" formData
scheme = byteString $ bool "http://" "https://" $ connectIsSecure ci scheme = byteString $ bool "http://" "https://" $ connectIsSecure ci
region = connectRegion ci region = connectRegion ci
url =
url = toS $ toLazyByteString $ scheme <> byteString (getHostAddr ci) <> toStrictBS $ toLazyByteString $
byteString "/" <> byteString (toS bucket) <> byteString "/" scheme <> byteString (getHostAddr ci)
<> byteString "/"
<> byteString bucket
<> byteString "/"
return (url, formData) return (url, formData)

View File

@ -15,12 +15,11 @@
-- --
module Network.Minio.PutObject module Network.Minio.PutObject
( ( putObjectInternal,
putObjectInternal ObjectData (..),
, ObjectData(..) selectPartSizes,
, selectPartSizes )
) where where
import Conduit (takeC) import Conduit (takeC)
import qualified Conduit as C import qualified Conduit as C
@ -29,16 +28,12 @@ import qualified Data.Conduit.Binary as CB
import qualified Data.Conduit.Combinators as CC import qualified Data.Conduit.Combinators as CC
import qualified Data.Conduit.List as CL import qualified Data.Conduit.List as CL
import qualified Data.List as List import qualified Data.List as List
import Lib.Prelude import Lib.Prelude
import Network.Minio.Data import Network.Minio.Data
import Network.Minio.Errors import Network.Minio.Errors
import Network.Minio.S3API import Network.Minio.S3API
import Network.Minio.Utils import Network.Minio.Utils
-- | A data-type to represent the source data for an object. A -- | A data-type to represent the source data for an object. A
-- file-path or a producer-conduit may be provided. -- file-path or a producer-conduit may be provided.
-- --
@ -50,36 +45,44 @@ import Network.Minio.Utils
-- the input - if it is not provided, upload will continue until the -- the input - if it is not provided, upload will continue until the
-- stream ends or the object reaches `maxObjectSize` size. -- stream ends or the object reaches `maxObjectSize` size.
data ObjectData m data ObjectData m
= ODFile FilePath (Maybe Int64) -- ^ Takes filepath and optional = -- | Takes filepath and optional
-- size. -- size.
| ODStream (C.ConduitM () ByteString m ()) (Maybe Int64) -- ^ Pass ODFile FilePath (Maybe Int64)
| -- | Pass
-- size -- size
-- (bytes) -- (bytes)
-- if -- if
-- known. -- known.
ODStream (C.ConduitM () ByteString m ()) (Maybe Int64)
-- | Put an object from ObjectData. This high-level API handles -- | Put an object from ObjectData. This high-level API handles
-- objects of all sizes, and even if the object size is unknown. -- objects of all sizes, and even if the object size is unknown.
putObjectInternal :: Bucket -> Object -> PutObjectOptions putObjectInternal ::
-> ObjectData Minio -> Minio ETag Bucket ->
Object ->
PutObjectOptions ->
ObjectData Minio ->
Minio ETag
putObjectInternal b o opts (ODStream src sizeMay) = do putObjectInternal b o opts (ODStream src sizeMay) = do
case sizeMay of case sizeMay of
-- unable to get size, so assume non-seekable file -- unable to get size, so assume non-seekable file
Nothing -> sequentialMultipartUpload b o opts Nothing src Nothing -> sequentialMultipartUpload b o opts Nothing src
-- got file size, so check for single/multipart upload -- got file size, so check for single/multipart upload
Just size -> Just size ->
if | size <= 64 * oneMiB -> do if
| size <= 64 * oneMiB -> do
bs <- C.runConduit $ src C..| takeC (fromIntegral size) C..| CB.sinkLbs bs <- C.runConduit $ src C..| takeC (fromIntegral size) C..| CB.sinkLbs
putObjectSingle' b o (pooToHeaders opts) $ LBS.toStrict bs putObjectSingle' b o (pooToHeaders opts) $ LBS.toStrict bs
| size > maxObjectSize -> throwIO $ MErrVPutSizeExceeded size | size > maxObjectSize -> throwIO $ MErrVPutSizeExceeded size
| otherwise -> sequentialMultipartUpload b o opts (Just size) src | otherwise -> sequentialMultipartUpload b o opts (Just size) src
putObjectInternal b o opts (ODFile fp sizeMay) = do putObjectInternal b o opts (ODFile fp sizeMay) = do
hResE <- withNewHandle fp $ \h -> hResE <- withNewHandle fp $ \h ->
liftM2 (,) (isHandleSeekable h) (getFileSize h) liftM2 (,) (isHandleSeekable h) (getFileSize h)
(isSeekable, handleSizeMay) <- either (const $ return (False, Nothing)) return (isSeekable, handleSizeMay) <-
either
(const $ return (False, Nothing))
return
hResE hResE
-- prefer given size to queried size. -- prefer given size to queried size.
@ -88,18 +91,25 @@ putObjectInternal b o opts (ODFile fp sizeMay) = do
case finalSizeMay of case finalSizeMay of
-- unable to get size, so assume non-seekable file -- unable to get size, so assume non-seekable file
Nothing -> sequentialMultipartUpload b o opts Nothing $ CB.sourceFile fp Nothing -> sequentialMultipartUpload b o opts Nothing $ CB.sourceFile fp
-- got file size, so check for single/multipart upload -- got file size, so check for single/multipart upload
Just size -> Just size ->
if | size <= 64 * oneMiB -> either throwIO return =<< if
withNewHandle fp (\h -> putObjectSingle b o (pooToHeaders opts) h 0 size) | size <= 64 * oneMiB ->
either throwIO return
=<< withNewHandle fp (\h -> putObjectSingle b o (pooToHeaders opts) h 0 size)
| size > maxObjectSize -> throwIO $ MErrVPutSizeExceeded size | size > maxObjectSize -> throwIO $ MErrVPutSizeExceeded size
| isSeekable -> parallelMultipartUpload b o opts fp size | isSeekable -> parallelMultipartUpload b o opts fp size
| otherwise -> sequentialMultipartUpload b o opts (Just size) $ | otherwise ->
sequentialMultipartUpload b o opts (Just size) $
CB.sourceFile fp CB.sourceFile fp
parallelMultipartUpload :: Bucket -> Object -> PutObjectOptions parallelMultipartUpload ::
-> FilePath -> Int64 -> Minio ETag Bucket ->
Object ->
PutObjectOptions ->
FilePath ->
Int64 ->
Minio ETag
parallelMultipartUpload b o opts filePath size = do parallelMultipartUpload b o opts filePath size = do
-- get a new upload id. -- get a new upload id.
uploadId <- newMultipartUpload b o (pooToHeaders opts) uploadId <- newMultipartUpload b o (pooToHeaders opts)
@ -109,15 +119,17 @@ parallelMultipartUpload b o opts filePath size = do
let threads = fromMaybe 10 $ pooNumThreads opts let threads = fromMaybe 10 $ pooNumThreads opts
-- perform upload with 'threads' threads -- perform upload with 'threads' threads
uploadedPartsE <- limitedMapConcurrently (fromIntegral threads) uploadedPartsE <-
(uploadPart uploadId) partSizeInfo limitedMapConcurrently
(fromIntegral threads)
(uploadPart uploadId)
partSizeInfo
-- if there were any errors, rethrow exception. -- if there were any errors, rethrow exception.
mapM_ throwIO $ lefts uploadedPartsE mapM_ throwIO $ lefts uploadedPartsE
-- if we get here, all parts were successfully uploaded. -- if we get here, all parts were successfully uploaded.
completeMultipartUpload b o uploadId $ rights uploadedPartsE completeMultipartUpload b o uploadId $ rights uploadedPartsE
where where
uploadPart uploadId (partNum, offset, sz) = uploadPart uploadId (partNum, offset, sz) =
withNewHandle filePath $ \h -> do withNewHandle filePath $ \h -> do
@ -125,10 +137,13 @@ parallelMultipartUpload b o opts filePath size = do
putObjectPart b o uploadId partNum [] payload putObjectPart b o uploadId partNum [] payload
-- | Upload multipart object from conduit source sequentially -- | Upload multipart object from conduit source sequentially
sequentialMultipartUpload :: Bucket -> Object -> PutObjectOptions sequentialMultipartUpload ::
-> Maybe Int64 Bucket ->
-> C.ConduitM () ByteString Minio () Object ->
-> Minio ETag PutObjectOptions ->
Maybe Int64 ->
C.ConduitM () ByteString Minio () ->
Minio ETag
sequentialMultipartUpload b o opts sizeMay src = do sequentialMultipartUpload b o opts sizeMay src = do
-- get a new upload id. -- get a new upload id.
uploadId <- newMultipartUpload b o (pooToHeaders opts) uploadId <- newMultipartUpload b o (pooToHeaders opts)
@ -136,8 +151,9 @@ sequentialMultipartUpload b o opts sizeMay src = do
-- upload parts in loop -- upload parts in loop
let partSizes = selectPartSizes $ maybe maxObjectSize identity sizeMay let partSizes = selectPartSizes $ maybe maxObjectSize identity sizeMay
(pnums, _, sizes) = List.unzip3 partSizes (pnums, _, sizes) = List.unzip3 partSizes
uploadedParts <- C.runConduit uploadedParts <-
$ src C.runConduit $
src
C..| chunkBSConduit (map fromIntegral sizes) C..| chunkBSConduit (map fromIntegral sizes)
C..| CL.map PayloadBS C..| CL.map PayloadBS
C..| uploadPart' uploadId pnums C..| uploadPart' uploadId pnums
@ -145,13 +161,13 @@ sequentialMultipartUpload b o opts sizeMay src = do
-- complete multipart upload -- complete multipart upload
completeMultipartUpload b o uploadId uploadedParts completeMultipartUpload b o uploadId uploadedParts
where where
uploadPart' _ [] = return () uploadPart' _ [] = return ()
uploadPart' uid (pn:pns) = do uploadPart' uid (pn : pns) = do
payloadMay <- C.await payloadMay <- C.await
case payloadMay of case payloadMay of
Nothing -> return () Nothing -> return ()
Just payload -> do pinfo <- lift $ putObjectPart b o uid pn [] payload Just payload -> do
pinfo <- lift $ putObjectPart b o uid pn [] payload
C.yield pinfo C.yield pinfo
uploadPart' uid pns uploadPart' uid pns

View File

@ -15,91 +15,88 @@
-- --
module Network.Minio.S3API module Network.Minio.S3API
( ( Region,
Region getLocation,
, getLocation
-- * Listing buckets -- * Listing buckets
-------------------- --------------------
, getService getService,
-- * Listing objects -- * Listing objects
-------------------- --------------------
, ListObjectsResult(..) ListObjectsResult (..),
, ListObjectsV1Result(..) ListObjectsV1Result (..),
, listObjects' listObjects',
, listObjectsV1' listObjectsV1',
-- * Retrieving buckets -- * Retrieving buckets
, headBucket headBucket,
-- * Retrieving objects -- * Retrieving objects
----------------------- -----------------------
, getObject' getObject',
, headObject headObject,
-- * Creating buckets and objects -- * Creating buckets and objects
--------------------------------- ---------------------------------
, putBucket putBucket,
, ETag ETag,
, maxSinglePutObjectSizeBytes maxSinglePutObjectSizeBytes,
, putObjectSingle' putObjectSingle',
, putObjectSingle putObjectSingle,
, copyObjectSingle copyObjectSingle,
-- * Multipart Upload APIs -- * Multipart Upload APIs
-------------------------- --------------------------
, UploadId UploadId,
, PartTuple PartTuple,
, Payload(..) Payload (..),
, PartNumber PartNumber,
, newMultipartUpload newMultipartUpload,
, putObjectPart putObjectPart,
, copyObjectPart copyObjectPart,
, completeMultipartUpload completeMultipartUpload,
, abortMultipartUpload abortMultipartUpload,
, ListUploadsResult(..) ListUploadsResult (..),
, listIncompleteUploads' listIncompleteUploads',
, ListPartsResult(..) ListPartsResult (..),
, listIncompleteParts' listIncompleteParts',
-- * Deletion APIs -- * Deletion APIs
-------------------------- --------------------------
, deleteBucket deleteBucket,
, deleteObject deleteObject,
-- * Presigned Operations -- * Presigned Operations
----------------------------- -----------------------------
, module Network.Minio.PresignedOperations module Network.Minio.PresignedOperations,
-- ** Bucket Policies -- ** Bucket Policies
, getBucketPolicy getBucketPolicy,
, setBucketPolicy setBucketPolicy,
-- * Bucket Notifications -- * Bucket Notifications
------------------------- -------------------------
, Notification(..) Notification (..),
, NotificationConfig(..) NotificationConfig (..),
, Arn Arn,
, Event(..) Event (..),
, Filter(..) Filter (..),
, FilterKey(..) FilterKey (..),
, FilterRules(..) FilterRules (..),
, FilterRule(..) FilterRule (..),
, getBucketNotification getBucketNotification,
, putBucketNotification putBucketNotification,
, removeAllBucketNotification removeAllBucketNotification,
) where )
where
import qualified Data.ByteString as BS import qualified Data.ByteString as BS
import qualified Data.Text as T import qualified Data.Text as T
import Lib.Prelude
import qualified Network.HTTP.Conduit as NC import qualified Network.HTTP.Conduit as NC
import qualified Network.HTTP.Types as HT import qualified Network.HTTP.Types as HT
import Network.HTTP.Types.Status (status404) import Network.HTTP.Types.Status (status404)
import UnliftIO (Handler (Handler))
import Lib.Prelude
import Network.Minio.API import Network.Minio.API
import Network.Minio.APICommon import Network.Minio.APICommon
import Network.Minio.Data import Network.Minio.Data
@ -108,12 +105,15 @@ import Network.Minio.PresignedOperations
import Network.Minio.Utils import Network.Minio.Utils
import Network.Minio.XmlGenerator import Network.Minio.XmlGenerator
import Network.Minio.XmlParser import Network.Minio.XmlParser
import UnliftIO (Handler (Handler))
-- | Fetch all buckets from the service. -- | Fetch all buckets from the service.
getService :: Minio [BucketInfo] getService :: Minio [BucketInfo]
getService = do getService = do
resp <- executeRequest $ defaultS3ReqInfo { resp <-
riNeedsLocation = False executeRequest $
defaultS3ReqInfo
{ riNeedsLocation = False
} }
parseListBuckets $ NC.responseBody resp parseListBuckets $ NC.responseBody resp
@ -132,21 +132,32 @@ parseGetObjectHeaders object headers =
-- | GET an object from the service and return parsed ObjectInfo and a -- | GET an object from the service and return parsed ObjectInfo and a
-- conduit source for the object content -- conduit source for the object content
getObject' :: Bucket -> Object -> HT.Query -> [HT.Header] getObject' ::
-> Minio GetObjectResponse Bucket ->
Object ->
HT.Query ->
[HT.Header] ->
Minio GetObjectResponse
getObject' bucket object queryParams headers = do getObject' bucket object queryParams headers = do
resp <- mkStreamRequest reqInfo resp <- mkStreamRequest reqInfo
let objInfoMaybe = parseGetObjectHeaders object $ NC.responseHeaders resp let objInfoMaybe = parseGetObjectHeaders object $ NC.responseHeaders resp
objInfo <- maybe (throwIO MErrVInvalidObjectInfoResponse) return objInfo <-
maybe
(throwIO MErrVInvalidObjectInfoResponse)
return
objInfoMaybe objInfoMaybe
return $ GetObjectResponse { gorObjectInfo = objInfo return $
, gorObjectStream = NC.responseBody resp GetObjectResponse
{ gorObjectInfo = objInfo,
gorObjectStream = NC.responseBody resp
} }
where where
reqInfo = defaultS3ReqInfo { riBucket = Just bucket reqInfo =
, riObject = Just object defaultS3ReqInfo
, riQueryParams = queryParams { riBucket = Just bucket,
, riHeaders = headers riObject = Just object,
riQueryParams = queryParams,
riHeaders = headers
} }
-- | Creates a bucket via a PUT bucket call. -- | Creates a bucket via a PUT bucket call.
@ -154,10 +165,11 @@ putBucket :: Bucket -> Region -> Minio ()
putBucket bucket location = do putBucket bucket location = do
ns <- asks getSvcNamespace ns <- asks getSvcNamespace
void $ executeRequest $ void $ executeRequest $
defaultS3ReqInfo { riMethod = HT.methodPut defaultS3ReqInfo
, riBucket = Just bucket { riMethod = HT.methodPut,
, riPayload = PayloadBS $ mkCreateBucketConfig ns location riBucket = Just bucket,
, riNeedsLocation = False riPayload = PayloadBS $ mkCreateBucketConfig ns location,
riNeedsLocation = False
} }
-- | Single PUT object size. -- | Single PUT object size.
@ -172,204 +184,286 @@ putObjectSingle' :: Bucket -> Object -> [HT.Header] -> ByteString -> Minio ETag
putObjectSingle' bucket object headers bs = do putObjectSingle' bucket object headers bs = do
let size = fromIntegral (BS.length bs) let size = fromIntegral (BS.length bs)
-- check length is within single PUT object size. -- check length is within single PUT object size.
when (size > maxSinglePutObjectSizeBytes) $ when (size > maxSinglePutObjectSizeBytes)
throwIO $ MErrVSinglePUTSizeExceeded size $ throwIO
$ MErrVSinglePUTSizeExceeded size
let payload = mkStreamingPayload $ PayloadBS bs let payload = mkStreamingPayload $ PayloadBS bs
resp <- executeRequest $ resp <-
defaultS3ReqInfo { riMethod = HT.methodPut executeRequest $
, riBucket = Just bucket defaultS3ReqInfo
, riObject = Just object { riMethod = HT.methodPut,
, riHeaders = headers riBucket = Just bucket,
, riPayload = payload riObject = Just object,
riHeaders = headers,
riPayload = payload
} }
let rheaders = NC.responseHeaders resp let rheaders = NC.responseHeaders resp
etag = getETagHeader rheaders etag = getETagHeader rheaders
maybe maybe
(throwIO MErrVETagHeaderNotFound) (throwIO MErrVETagHeaderNotFound)
return etag return
etag
-- | PUT an object into the service. This function performs a single -- | PUT an object into the service. This function performs a single
-- PUT object call, and so can only transfer objects upto 5GiB. -- PUT object call, and so can only transfer objects upto 5GiB.
putObjectSingle :: Bucket -> Object -> [HT.Header] -> Handle -> Int64 putObjectSingle ::
-> Int64 -> Minio ETag Bucket ->
Object ->
[HT.Header] ->
Handle ->
Int64 ->
Int64 ->
Minio ETag
putObjectSingle bucket object headers h offset size = do putObjectSingle bucket object headers h offset size = do
-- check length is within single PUT object size. -- check length is within single PUT object size.
when (size > maxSinglePutObjectSizeBytes) $ when (size > maxSinglePutObjectSizeBytes)
throwIO $ MErrVSinglePUTSizeExceeded size $ throwIO
$ MErrVSinglePUTSizeExceeded size
-- content-length header is automatically set by library. -- content-length header is automatically set by library.
let payload = mkStreamingPayload $ PayloadH h offset size let payload = mkStreamingPayload $ PayloadH h offset size
resp <- executeRequest $ resp <-
defaultS3ReqInfo { riMethod = HT.methodPut executeRequest $
, riBucket = Just bucket defaultS3ReqInfo
, riObject = Just object { riMethod = HT.methodPut,
, riHeaders = headers riBucket = Just bucket,
, riPayload = payload riObject = Just object,
riHeaders = headers,
riPayload = payload
} }
let rheaders = NC.responseHeaders resp let rheaders = NC.responseHeaders resp
etag = getETagHeader rheaders etag = getETagHeader rheaders
maybe maybe
(throwIO MErrVETagHeaderNotFound) (throwIO MErrVETagHeaderNotFound)
return etag return
etag
-- | List objects in a bucket matching prefix up to delimiter, -- | List objects in a bucket matching prefix up to delimiter,
-- starting from nextMarker. -- starting from nextMarker.
listObjectsV1' :: Bucket -> Maybe Text -> Maybe Text -> Maybe Text -> Maybe Int listObjectsV1' ::
-> Minio ListObjectsV1Result Bucket ->
Maybe Text ->
Maybe Text ->
Maybe Text ->
Maybe Int ->
Minio ListObjectsV1Result
listObjectsV1' bucket prefix nextMarker delimiter maxKeys = do listObjectsV1' bucket prefix nextMarker delimiter maxKeys = do
resp <- executeRequest $ defaultS3ReqInfo { riMethod = HT.methodGet resp <-
, riBucket = Just bucket executeRequest $
, riQueryParams = mkOptionalParams params defaultS3ReqInfo
{ riMethod = HT.methodGet,
riBucket = Just bucket,
riQueryParams = mkOptionalParams params
} }
parseListObjectsV1Response $ NC.responseBody resp parseListObjectsV1Response $ NC.responseBody resp
where where
params = [ params =
("marker", nextMarker) [ ("marker", nextMarker),
, ("prefix", prefix) ("prefix", prefix),
, ("delimiter", delimiter) ("delimiter", delimiter),
, ("max-keys", show <$> maxKeys) ("max-keys", show <$> maxKeys)
] ]
-- | List objects in a bucket matching prefix up to delimiter, -- | List objects in a bucket matching prefix up to delimiter,
-- starting from nextToken. -- starting from nextToken.
listObjects' :: Bucket -> Maybe Text -> Maybe Text -> Maybe Text -> Maybe Int listObjects' ::
-> Minio ListObjectsResult Bucket ->
Maybe Text ->
Maybe Text ->
Maybe Text ->
Maybe Int ->
Minio ListObjectsResult
listObjects' bucket prefix nextToken delimiter maxKeys = do listObjects' bucket prefix nextToken delimiter maxKeys = do
resp <- executeRequest $ defaultS3ReqInfo { riMethod = HT.methodGet resp <-
, riBucket = Just bucket executeRequest $
, riQueryParams = mkOptionalParams params defaultS3ReqInfo
{ riMethod = HT.methodGet,
riBucket = Just bucket,
riQueryParams = mkOptionalParams params
} }
parseListObjectsResponse $ NC.responseBody resp parseListObjectsResponse $ NC.responseBody resp
where where
params = [ params =
("list-type", Just "2") [ ("list-type", Just "2"),
, ("continuation_token", nextToken) ("continuation_token", nextToken),
, ("prefix", prefix) ("prefix", prefix),
, ("delimiter", delimiter) ("delimiter", delimiter),
, ("max-keys", show <$> maxKeys) ("max-keys", show <$> maxKeys)
] ]
-- | DELETE a bucket from the service. -- | DELETE a bucket from the service.
deleteBucket :: Bucket -> Minio () deleteBucket :: Bucket -> Minio ()
deleteBucket bucket = void $ deleteBucket bucket =
executeRequest $ void
defaultS3ReqInfo { riMethod = HT.methodDelete $ executeRequest
, riBucket = Just bucket $ defaultS3ReqInfo
{ riMethod = HT.methodDelete,
riBucket = Just bucket
} }
-- | DELETE an object from the service. -- | DELETE an object from the service.
deleteObject :: Bucket -> Object -> Minio () deleteObject :: Bucket -> Object -> Minio ()
deleteObject bucket object = void $ deleteObject bucket object =
executeRequest $ void
defaultS3ReqInfo { riMethod = HT.methodDelete $ executeRequest
, riBucket = Just bucket $ defaultS3ReqInfo
, riObject = Just object { riMethod = HT.methodDelete,
riBucket = Just bucket,
riObject = Just object
} }
-- | Create a new multipart upload. -- | Create a new multipart upload.
newMultipartUpload :: Bucket -> Object -> [HT.Header] -> Minio UploadId newMultipartUpload :: Bucket -> Object -> [HT.Header] -> Minio UploadId
newMultipartUpload bucket object headers = do newMultipartUpload bucket object headers = do
resp <- executeRequest $ defaultS3ReqInfo { riMethod = HT.methodPost resp <-
, riBucket = Just bucket executeRequest $
, riObject = Just object defaultS3ReqInfo
, riQueryParams = [("uploads", Nothing)] { riMethod = HT.methodPost,
, riHeaders = headers riBucket = Just bucket,
riObject = Just object,
riQueryParams = [("uploads", Nothing)],
riHeaders = headers
} }
parseNewMultipartUpload $ NC.responseBody resp parseNewMultipartUpload $ NC.responseBody resp
-- | PUT a part of an object as part of a multipart upload. -- | PUT a part of an object as part of a multipart upload.
putObjectPart :: Bucket -> Object -> UploadId -> PartNumber -> [HT.Header] putObjectPart ::
-> Payload -> Minio PartTuple Bucket ->
Object ->
UploadId ->
PartNumber ->
[HT.Header] ->
Payload ->
Minio PartTuple
putObjectPart bucket object uploadId partNumber headers payload = do putObjectPart bucket object uploadId partNumber headers payload = do
-- transform payload to conduit to enable streaming signature -- transform payload to conduit to enable streaming signature
let payload' = mkStreamingPayload payload let payload' = mkStreamingPayload payload
resp <- executeRequest $ resp <-
defaultS3ReqInfo { riMethod = HT.methodPut executeRequest $
, riBucket = Just bucket defaultS3ReqInfo
, riObject = Just object { riMethod = HT.methodPut,
, riQueryParams = mkOptionalParams params riBucket = Just bucket,
, riHeaders = headers riObject = Just object,
, riPayload = payload' riQueryParams = mkOptionalParams params,
riHeaders = headers,
riPayload = payload'
} }
let rheaders = NC.responseHeaders resp let rheaders = NC.responseHeaders resp
etag = getETagHeader rheaders etag = getETagHeader rheaders
maybe maybe
(throwIO MErrVETagHeaderNotFound) (throwIO MErrVETagHeaderNotFound)
(return . (partNumber, )) etag (return . (partNumber,))
etag
where where
params = [ params =
("uploadId", Just uploadId) [ ("uploadId", Just uploadId),
, ("partNumber", Just $ show partNumber) ("partNumber", Just $ show partNumber)
] ]
srcInfoToHeaders :: SourceInfo -> [HT.Header] srcInfoToHeaders :: SourceInfo -> [HT.Header]
srcInfoToHeaders srcInfo = ("x-amz-copy-source", srcInfoToHeaders srcInfo =
toS $ T.concat ["/", srcBucket srcInfo, ( "x-amz-copy-source",
"/", srcObject srcInfo] toUtf8 $
) : rangeHdr ++ zip names values T.concat
[ "/",
srcBucket srcInfo,
"/",
srcObject srcInfo
]
)
: rangeHdr
++ zip names values
where where
names = ["x-amz-copy-source-if-match", "x-amz-copy-source-if-none-match", names =
[ "x-amz-copy-source-if-match",
"x-amz-copy-source-if-none-match",
"x-amz-copy-source-if-unmodified-since", "x-amz-copy-source-if-unmodified-since",
"x-amz-copy-source-if-modified-since"] "x-amz-copy-source-if-modified-since"
values = mapMaybe (fmap encodeUtf8 . (srcInfo &)) ]
[srcIfMatch, srcIfNoneMatch, values =
mapMaybe
(fmap encodeUtf8 . (srcInfo &))
[ srcIfMatch,
srcIfNoneMatch,
fmap formatRFC1123 . srcIfUnmodifiedSince, fmap formatRFC1123 . srcIfUnmodifiedSince,
fmap formatRFC1123 . srcIfModifiedSince] fmap formatRFC1123 . srcIfModifiedSince
rangeHdr = maybe [] (\a -> [("x-amz-copy-source-range", HT.renderByteRanges [a])]) ]
$ toByteRange <$> srcRange srcInfo rangeHdr =
maybe [] (\a -> [("x-amz-copy-source-range", HT.renderByteRanges [a])]) $
toByteRange <$> srcRange srcInfo
toByteRange :: (Int64, Int64) -> HT.ByteRange toByteRange :: (Int64, Int64) -> HT.ByteRange
toByteRange (x, y) = HT.ByteRangeFromTo (fromIntegral x) (fromIntegral y) toByteRange (x, y) = HT.ByteRangeFromTo (fromIntegral x) (fromIntegral y)
-- | Performs server-side copy of an object or part of an object as an -- | Performs server-side copy of an object or part of an object as an
-- upload part of an ongoing multi-part upload. -- upload part of an ongoing multi-part upload.
copyObjectPart :: DestinationInfo -> SourceInfo -> UploadId copyObjectPart ::
-> PartNumber -> [HT.Header] -> Minio (ETag, UTCTime) DestinationInfo ->
SourceInfo ->
UploadId ->
PartNumber ->
[HT.Header] ->
Minio (ETag, UTCTime)
copyObjectPart dstInfo srcInfo uploadId partNumber headers = do copyObjectPart dstInfo srcInfo uploadId partNumber headers = do
resp <- executeRequest $ resp <-
defaultS3ReqInfo { riMethod = HT.methodPut executeRequest $
, riBucket = Just $ dstBucket dstInfo defaultS3ReqInfo
, riObject = Just $ dstObject dstInfo { riMethod = HT.methodPut,
, riQueryParams = mkOptionalParams params riBucket = Just $ dstBucket dstInfo,
, riHeaders = headers ++ srcInfoToHeaders srcInfo riObject = Just $ dstObject dstInfo,
riQueryParams = mkOptionalParams params,
riHeaders = headers ++ srcInfoToHeaders srcInfo
} }
parseCopyObjectResponse $ NC.responseBody resp parseCopyObjectResponse $ NC.responseBody resp
where where
params = [ params =
("uploadId", Just uploadId) [ ("uploadId", Just uploadId),
, ("partNumber", Just $ show partNumber) ("partNumber", Just $ show partNumber)
] ]
-- | Performs server-side copy of an object that is upto 5GiB in -- | Performs server-side copy of an object that is upto 5GiB in
-- size. If the object is greater than 5GiB, this function throws the -- size. If the object is greater than 5GiB, this function throws the
-- error returned by the server. -- error returned by the server.
copyObjectSingle :: Bucket -> Object -> SourceInfo -> [HT.Header] copyObjectSingle ::
-> Minio (ETag, UTCTime) Bucket ->
Object ->
SourceInfo ->
[HT.Header] ->
Minio (ETag, UTCTime)
copyObjectSingle bucket object srcInfo headers = do copyObjectSingle bucket object srcInfo headers = do
-- validate that srcRange is Nothing for this API. -- validate that srcRange is Nothing for this API.
when (isJust $ srcRange srcInfo) $ when (isJust $ srcRange srcInfo) $
throwIO MErrVCopyObjSingleNoRangeAccepted throwIO MErrVCopyObjSingleNoRangeAccepted
resp <- executeRequest $ resp <-
defaultS3ReqInfo { riMethod = HT.methodPut executeRequest $
, riBucket = Just bucket defaultS3ReqInfo
, riObject = Just object { riMethod = HT.methodPut,
, riHeaders = headers ++ srcInfoToHeaders srcInfo riBucket = Just bucket,
riObject = Just object,
riHeaders = headers ++ srcInfoToHeaders srcInfo
} }
parseCopyObjectResponse $ NC.responseBody resp parseCopyObjectResponse $ NC.responseBody resp
-- | Complete a multipart upload. -- | Complete a multipart upload.
completeMultipartUpload :: Bucket -> Object -> UploadId -> [PartTuple] completeMultipartUpload ::
-> Minio ETag Bucket ->
Object ->
UploadId ->
[PartTuple] ->
Minio ETag
completeMultipartUpload bucket object uploadId partTuple = do completeMultipartUpload bucket object uploadId partTuple = do
resp <- executeRequest $ resp <-
defaultS3ReqInfo { riMethod = HT.methodPost executeRequest $
, riBucket = Just bucket defaultS3ReqInfo
, riObject = Just object { riMethod = HT.methodPost,
, riQueryParams = mkOptionalParams params riBucket = Just bucket,
, riPayload = PayloadBS $ riObject = Just object,
riQueryParams = mkOptionalParams params,
riPayload =
PayloadBS $
mkCompleteMultipartUploadRequest partTuple mkCompleteMultipartUploadRequest partTuple
} }
parseCompleteMultipartUploadResponse $ NC.responseBody resp parseCompleteMultipartUploadResponse $ NC.responseBody resp
@ -378,88 +472,114 @@ completeMultipartUpload bucket object uploadId partTuple = do
-- | Abort a multipart upload. -- | Abort a multipart upload.
abortMultipartUpload :: Bucket -> Object -> UploadId -> Minio () abortMultipartUpload :: Bucket -> Object -> UploadId -> Minio ()
abortMultipartUpload bucket object uploadId = void $ abortMultipartUpload bucket object uploadId =
executeRequest $ defaultS3ReqInfo { riMethod = HT.methodDelete void
, riBucket = Just bucket $ executeRequest
, riObject = Just object $ defaultS3ReqInfo
, riQueryParams = mkOptionalParams params { riMethod = HT.methodDelete,
riBucket = Just bucket,
riObject = Just object,
riQueryParams = mkOptionalParams params
} }
where where
params = [("uploadId", Just uploadId)] params = [("uploadId", Just uploadId)]
-- | List incomplete multipart uploads. -- | List incomplete multipart uploads.
listIncompleteUploads' :: Bucket -> Maybe Text -> Maybe Text -> Maybe Text listIncompleteUploads' ::
-> Maybe Text -> Maybe Int -> Minio ListUploadsResult Bucket ->
Maybe Text ->
Maybe Text ->
Maybe Text ->
Maybe Text ->
Maybe Int ->
Minio ListUploadsResult
listIncompleteUploads' bucket prefix delimiter keyMarker uploadIdMarker maxKeys = do listIncompleteUploads' bucket prefix delimiter keyMarker uploadIdMarker maxKeys = do
resp <- executeRequest $ defaultS3ReqInfo { riMethod = HT.methodGet resp <-
, riBucket = Just bucket executeRequest $
, riQueryParams = params defaultS3ReqInfo
{ riMethod = HT.methodGet,
riBucket = Just bucket,
riQueryParams = params
} }
parseListUploadsResponse $ NC.responseBody resp parseListUploadsResponse $ NC.responseBody resp
where where
-- build query params -- build query params
params = ("uploads", Nothing) : mkOptionalParams params =
[ ("prefix", prefix) ("uploads", Nothing)
, ("delimiter", delimiter) : mkOptionalParams
, ("key-marker", keyMarker) [ ("prefix", prefix),
, ("upload-id-marker", uploadIdMarker) ("delimiter", delimiter),
, ("max-uploads", show <$> maxKeys) ("key-marker", keyMarker),
("upload-id-marker", uploadIdMarker),
("max-uploads", show <$> maxKeys)
] ]
-- | List parts of an ongoing multipart upload. -- | List parts of an ongoing multipart upload.
listIncompleteParts' :: Bucket -> Object -> UploadId -> Maybe Text listIncompleteParts' ::
-> Maybe Text -> Minio ListPartsResult Bucket ->
Object ->
UploadId ->
Maybe Text ->
Maybe Text ->
Minio ListPartsResult
listIncompleteParts' bucket object uploadId maxParts partNumMarker = do listIncompleteParts' bucket object uploadId maxParts partNumMarker = do
resp <- executeRequest $ defaultS3ReqInfo { riMethod = HT.methodGet resp <-
, riBucket = Just bucket executeRequest $
, riObject = Just object defaultS3ReqInfo
, riQueryParams = mkOptionalParams params { riMethod = HT.methodGet,
riBucket = Just bucket,
riObject = Just object,
riQueryParams = mkOptionalParams params
} }
parseListPartsResponse $ NC.responseBody resp parseListPartsResponse $ NC.responseBody resp
where where
-- build optional query params -- build optional query params
params = [ params =
("uploadId", Just uploadId) [ ("uploadId", Just uploadId),
, ("part-number-marker", partNumMarker) ("part-number-marker", partNumMarker),
, ("max-parts", maxParts) ("max-parts", maxParts)
] ]
-- | Get metadata of an object. -- | Get metadata of an object.
headObject :: Bucket -> Object -> [HT.Header] -> Minio ObjectInfo headObject :: Bucket -> Object -> [HT.Header] -> Minio ObjectInfo
headObject bucket object reqHeaders = do headObject bucket object reqHeaders = do
resp <- executeRequest $ defaultS3ReqInfo { riMethod = HT.methodHead resp <-
, riBucket = Just bucket executeRequest $
, riObject = Just object defaultS3ReqInfo
, riHeaders = reqHeaders { riMethod = HT.methodHead,
riBucket = Just bucket,
riObject = Just object,
riHeaders = reqHeaders
} }
maybe (throwIO MErrVInvalidObjectInfoResponse) return $ maybe (throwIO MErrVInvalidObjectInfoResponse) return
parseGetObjectHeaders object $ NC.responseHeaders resp $ parseGetObjectHeaders object
$ NC.responseHeaders resp
-- | Query the object store if a given bucket exists. -- | Query the object store if a given bucket exists.
headBucket :: Bucket -> Minio Bool headBucket :: Bucket -> Minio Bool
headBucket bucket = headBucketEx `catches` headBucket bucket =
[ Handler handleNoSuchBucket headBucketEx
, Handler handleStatus404 `catches` [ Handler handleNoSuchBucket,
Handler handleStatus404
] ]
where where
handleNoSuchBucket :: ServiceErr -> Minio Bool handleNoSuchBucket :: ServiceErr -> Minio Bool
handleNoSuchBucket e | e == NoSuchBucket = return False handleNoSuchBucket e
| e == NoSuchBucket = return False
| otherwise = throwIO e | otherwise = throwIO e
handleStatus404 :: NC.HttpException -> Minio Bool handleStatus404 :: NC.HttpException -> Minio Bool
handleStatus404 e@(NC.HttpExceptionRequest _ (NC.StatusCodeException res _)) = handleStatus404 e@(NC.HttpExceptionRequest _ (NC.StatusCodeException res _)) =
if NC.responseStatus res == status404 if NC.responseStatus res == status404
then return False then return False
else throwIO e else throwIO e
handleStatus404 e = throwIO e handleStatus404 e = throwIO e
headBucketEx = do headBucketEx = do
resp <- executeRequest $ defaultS3ReqInfo { riMethod = HT.methodHead resp <-
, riBucket = Just bucket executeRequest $
defaultS3ReqInfo
{ riMethod = HT.methodHead,
riBucket = Just bucket
} }
return $ NC.responseStatus resp == HT.ok200 return $ NC.responseStatus resp == HT.ok200
@ -467,19 +587,25 @@ headBucket bucket = headBucketEx `catches`
putBucketNotification :: Bucket -> Notification -> Minio () putBucketNotification :: Bucket -> Notification -> Minio ()
putBucketNotification bucket ncfg = do putBucketNotification bucket ncfg = do
ns <- asks getSvcNamespace ns <- asks getSvcNamespace
void $ executeRequest $ defaultS3ReqInfo { riMethod = HT.methodPut void $ executeRequest $
, riBucket = Just bucket defaultS3ReqInfo
, riQueryParams = [("notification", Nothing)] { riMethod = HT.methodPut,
, riPayload = PayloadBS $ riBucket = Just bucket,
riQueryParams = [("notification", Nothing)],
riPayload =
PayloadBS $
mkPutNotificationRequest ns ncfg mkPutNotificationRequest ns ncfg
} }
-- | Retrieve the notification configuration on a bucket. -- | Retrieve the notification configuration on a bucket.
getBucketNotification :: Bucket -> Minio Notification getBucketNotification :: Bucket -> Minio Notification
getBucketNotification bucket = do getBucketNotification bucket = do
resp <- executeRequest $ defaultS3ReqInfo { riMethod = HT.methodGet resp <-
, riBucket = Just bucket executeRequest $
, riQueryParams = [("notification", Nothing)] defaultS3ReqInfo
{ riMethod = HT.methodGet,
riBucket = Just bucket,
riQueryParams = [("notification", Nothing)]
} }
parseNotification $ NC.responseBody resp parseNotification $ NC.responseBody resp
@ -490,11 +616,14 @@ removeAllBucketNotification = flip putBucketNotification defaultNotification
-- | Fetch the policy if any on a bucket. -- | Fetch the policy if any on a bucket.
getBucketPolicy :: Bucket -> Minio Text getBucketPolicy :: Bucket -> Minio Text
getBucketPolicy bucket = do getBucketPolicy bucket = do
resp <- executeRequest $ defaultS3ReqInfo { riMethod = HT.methodGet resp <-
, riBucket = Just bucket executeRequest $
, riQueryParams = [("policy", Nothing)] defaultS3ReqInfo
{ riMethod = HT.methodGet,
riBucket = Just bucket,
riQueryParams = [("policy", Nothing)]
} }
return $ toS $ NC.responseBody resp return $ decodeUtf8Lenient $ toStrictBS $ NC.responseBody resp
-- | Set a new policy on a bucket. -- | Set a new policy on a bucket.
-- As a special condition if the policy is empty -- As a special condition if the policy is empty
@ -506,18 +635,22 @@ setBucketPolicy bucket policy = do
else putBucketPolicy bucket policy else putBucketPolicy bucket policy
-- | Save a new policy on a bucket. -- | Save a new policy on a bucket.
putBucketPolicy :: Bucket -> Text -> Minio() putBucketPolicy :: Bucket -> Text -> Minio ()
putBucketPolicy bucket policy = do putBucketPolicy bucket policy = do
void $ executeRequest $ defaultS3ReqInfo { riMethod = HT.methodPut void $ executeRequest $
, riBucket = Just bucket defaultS3ReqInfo
, riQueryParams = [("policy", Nothing)] { riMethod = HT.methodPut,
, riPayload = PayloadBS $ encodeUtf8 policy riBucket = Just bucket,
riQueryParams = [("policy", Nothing)],
riPayload = PayloadBS $ encodeUtf8 policy
} }
-- | Delete any policy set on a bucket. -- | Delete any policy set on a bucket.
deleteBucketPolicy :: Bucket -> Minio() deleteBucketPolicy :: Bucket -> Minio ()
deleteBucketPolicy bucket = do deleteBucketPolicy bucket = do
void $ executeRequest $ defaultS3ReqInfo { riMethod = HT.methodDelete void $ executeRequest $
, riBucket = Just bucket defaultS3ReqInfo
, riQueryParams = [("policy", Nothing)] { riMethod = HT.methodDelete,
riBucket = Just bucket,
riQueryParams = [("policy", Nothing)]
} }

View File

@ -15,60 +15,51 @@
-- --
module Network.Minio.SelectAPI module Network.Minio.SelectAPI
( ( -- | The `selectObjectContent` allows querying CSV, JSON or Parquet
-- | The `selectObjectContent` allows querying CSV, JSON or Parquet
-- format objects in AWS S3 and in MinIO using SQL Select -- format objects in AWS S3 and in MinIO using SQL Select
-- statements. This allows significant reduction of data transfer -- statements. This allows significant reduction of data transfer
-- from object storage for computation-intensive tasks, as relevant -- from object storage for computation-intensive tasks, as relevant
-- data is filtered close to the storage. -- data is filtered close to the storage.
selectObjectContent,
selectObjectContent SelectRequest,
selectRequest,
, SelectRequest
, selectRequest
-- *** Input Serialization -- *** Input Serialization
InputSerialization,
, InputSerialization defaultCsvInput,
, defaultCsvInput linesJsonInput,
, linesJsonInput documentJsonInput,
, documentJsonInput defaultParquetInput,
, defaultParquetInput setInputCSVProps,
, setInputCSVProps CompressionType (..),
setInputCompressionType,
, CompressionType(..)
, setInputCompressionType
-- *** CSV Format details -- *** CSV Format details
-- | CSV format options such as delimiters and quote characters are -- | CSV format options such as delimiters and quote characters are
-- specified using using the functions below. Options are combined -- specified using using the functions below. Options are combined
-- monoidally. -- monoidally.
CSVProp,
, CSVProp recordDelimiter,
, recordDelimiter fieldDelimiter,
, fieldDelimiter quoteCharacter,
, quoteCharacter quoteEscapeCharacter,
, quoteEscapeCharacter commentCharacter,
, commentCharacter allowQuotedRecordDelimiter,
, allowQuotedRecordDelimiter FileHeaderInfo (..),
, FileHeaderInfo(..) fileHeaderInfo,
, fileHeaderInfo QuoteFields (..),
, QuoteFields(..) quoteFields,
, quoteFields
-- *** Output Serialization -- *** Output Serialization
OutputSerialization,
, OutputSerialization defaultCsvOutput,
, defaultCsvOutput defaultJsonOutput,
, defaultJsonOutput outputCSVFromProps,
, outputCSVFromProps outputJSONFromRecordDelimiter,
, outputJSONFromRecordDelimiter
-- *** Progress messages -- *** Progress messages
setRequestProgressEnabled,
, setRequestProgressEnabled
-- *** Interpreting Select output -- *** Interpreting Select output
@ -88,12 +79,12 @@ module Network.Minio.SelectAPI
-- --
-- Note that runConduit, the connect operator (.|) and stdoutC are -- Note that runConduit, the connect operator (.|) and stdoutC are
-- all from the "conduit" package. -- all from the "conduit" package.
getPayloadBytes,
, getPayloadBytes EventMessage (..),
, EventMessage(..) Progress (..),
, Progress(..) Stats,
, Stats )
) where where
import Conduit ((.|)) import Conduit ((.|))
import qualified Conduit as C import qualified Conduit as C
@ -101,20 +92,19 @@ import qualified Data.Binary as Bin
import qualified Data.ByteString as B import qualified Data.ByteString as B
import qualified Data.ByteString.Lazy as LB import qualified Data.ByteString.Lazy as LB
import Data.Digest.CRC32 (crc32, crc32Update) import Data.Digest.CRC32 (crc32, crc32Update)
import Lib.Prelude
import qualified Network.HTTP.Conduit as NC import qualified Network.HTTP.Conduit as NC
import qualified Network.HTTP.Types as HT import qualified Network.HTTP.Types as HT
import UnliftIO (MonadUnliftIO)
import Lib.Prelude
import Network.Minio.API import Network.Minio.API
import Network.Minio.Data import Network.Minio.Data
import Network.Minio.Errors import Network.Minio.Errors
import Network.Minio.Utils import Network.Minio.Utils
import Network.Minio.XmlGenerator import Network.Minio.XmlGenerator
import Network.Minio.XmlParser import Network.Minio.XmlParser
import UnliftIO (MonadUnliftIO)
data EventStreamException = ESEPreludeCRCFailed data EventStreamException
= ESEPreludeCRCFailed
| ESEMessageCRCFailed | ESEMessageCRCFailed
| ESEUnexpectedEndOfStream | ESEUnexpectedEndOfStream
| ESEDecodeFail [Char] | ESEDecodeFail [Char]
@ -144,8 +134,10 @@ bytesToHeaderName t = case t of
":error-message" -> return ErrorMessage ":error-message" -> return ErrorMessage
_ -> throwIO ESEInvalidHeaderType _ -> throwIO ESEInvalidHeaderType
parseHeaders :: MonadUnliftIO m parseHeaders ::
=> Word32 -> C.ConduitM ByteString a m [MessageHeader] MonadUnliftIO m =>
Word32 ->
C.ConduitM ByteString a m [MessageHeader]
parseHeaders 0 = return [] parseHeaders 0 = return []
parseHeaders hdrLen = do parseHeaders hdrLen = do
bs1 <- readNBytes 1 bs1 <- readNBytes 1
@ -167,7 +159,7 @@ parseHeaders hdrLen = do
k = 1 + fromIntegral n + 1 + 2 + fromIntegral vLen k = 1 + fromIntegral n + 1 + 2 + fromIntegral vLen
ms <- parseHeaders (hdrLen - k) ms <- parseHeaders (hdrLen - k)
return (m:ms) return (m : ms)
-- readNBytes returns N bytes read from the string and throws an -- readNBytes returns N bytes read from the string and throws an
-- exception if N bytes are not present on the stream. -- exception if N bytes are not present on the stream.
@ -178,8 +170,9 @@ readNBytes n = do
then throwIO ESEUnexpectedEndOfStream then throwIO ESEUnexpectedEndOfStream
else return b else return b
crcCheck :: MonadUnliftIO m crcCheck ::
=> C.ConduitM ByteString ByteString m () MonadUnliftIO m =>
C.ConduitM ByteString ByteString m ()
crcCheck = do crcCheck = do
b <- readNBytes 12 b <- readNBytes 12
n :: Word32 <- liftIO $ parseBinary $ B.take 4 b n :: Word32 <- liftIO $ parseBinary $ B.take 4 b
@ -193,7 +186,7 @@ crcCheck = do
-- 12 bytes have been read off the current message. Now read the -- 12 bytes have been read off the current message. Now read the
-- next (n-12)-4 bytes and accumulate the checksum, and yield it. -- next (n-12)-4 bytes and accumulate the checksum, and yield it.
let startCrc = crc32 b let startCrc = crc32 b
finalCrc <- accumulateYield (fromIntegral n-16) startCrc finalCrc <- accumulateYield (fromIntegral n -16) startCrc
bs <- readNBytes 4 bs <- readNBytes 4
expectedCrc :: Word32 <- liftIO $ parseBinary bs expectedCrc :: Word32 <- liftIO $ parseBinary bs
@ -248,13 +241,10 @@ handleMessage = do
Just "End" -> return () Just "End" -> return ()
_ -> throwIO ESEInvalidMessageType _ -> throwIO ESEInvalidMessageType
when (eventHdrValue /= Just "End") handleMessage when (eventHdrValue /= Just "End") handleMessage
Just "error" -> do Just "error" -> do
let reqMsgMay = RequestLevelErrorMessage <$> errCode <*> errMsg let reqMsgMay = RequestLevelErrorMessage <$> errCode <*> errMsg
maybe (throwIO ESEInvalidMessageType) C.yield reqMsgMay maybe (throwIO ESEInvalidMessageType) C.yield reqMsgMay
_ -> throwIO ESEInvalidMessageType _ -> throwIO ESEInvalidMessageType
where where
passThrough 0 = return () passThrough 0 = return ()
passThrough n = do passThrough n = do
@ -263,23 +253,28 @@ handleMessage = do
C.yield $ RecordPayloadEventMessage b C.yield $ RecordPayloadEventMessage b
passThrough $ n - B.length b passThrough $ n - B.length b
selectProtoConduit ::
selectProtoConduit :: MonadUnliftIO m MonadUnliftIO m =>
=> C.ConduitT ByteString EventMessage m () C.ConduitT ByteString EventMessage m ()
selectProtoConduit = crcCheck .| handleMessage selectProtoConduit = crcCheck .| handleMessage
-- | selectObjectContent calls the SelectRequest on the given -- | selectObjectContent calls the SelectRequest on the given
-- object. It returns a Conduit of event messages that can be consumed -- object. It returns a Conduit of event messages that can be consumed
-- by the client. -- by the client.
selectObjectContent :: Bucket -> Object -> SelectRequest selectObjectContent ::
-> Minio (C.ConduitT () EventMessage Minio ()) Bucket ->
Object ->
SelectRequest ->
Minio (C.ConduitT () EventMessage Minio ())
selectObjectContent b o r = do selectObjectContent b o r = do
let reqInfo = defaultS3ReqInfo { riMethod = HT.methodPost let reqInfo =
, riBucket = Just b defaultS3ReqInfo
, riObject = Just o { riMethod = HT.methodPost,
, riPayload = PayloadBS $ mkSelectRequest r riBucket = Just b,
, riNeedsLocation = False riObject = Just o,
, riQueryParams = [("select", Nothing), ("select-type", Just "2")] riPayload = PayloadBS $ mkSelectRequest r,
riNeedsLocation = False,
riQueryParams = [("select", Nothing), ("select-type", Just "2")]
} }
--print $ mkSelectRequest r --print $ mkSelectRequest r
resp <- mkStreamRequest reqInfo resp <- mkStreamRequest reqInfo

View File

@ -13,6 +13,7 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE BangPatterns #-}
module Network.Minio.Sign.V4 where module Network.Minio.Sign.V4 where
@ -20,50 +21,54 @@ import qualified Conduit as C
import qualified Data.ByteString as B import qualified Data.ByteString as B
import qualified Data.ByteString.Base64 as Base64 import qualified Data.ByteString.Base64 as Base64
import qualified Data.ByteString.Char8 as B8 import qualified Data.ByteString.Char8 as B8
import qualified Data.ByteString.Lazy as LB
import Data.CaseInsensitive (mk) import Data.CaseInsensitive (mk)
import qualified Data.CaseInsensitive as CI import qualified Data.CaseInsensitive as CI
import qualified Data.HashMap.Strict as Map import qualified Data.HashMap.Strict as Map
import qualified Data.HashSet as Set import qualified Data.HashSet as Set
import qualified Data.Time as Time import qualified Data.Time as Time
import Lib.Prelude
import qualified Network.HTTP.Conduit as NC import qualified Network.HTTP.Conduit as NC
import Network.HTTP.Types (Header, parseQuery) import Network.HTTP.Types (Header, parseQuery)
import qualified Network.HTTP.Types as H import qualified Network.HTTP.Types as H
import Text.Printf (printf)
import Lib.Prelude
import Network.Minio.Data.ByteString import Network.Minio.Data.ByteString
import Network.Minio.Data.Crypto import Network.Minio.Data.Crypto
import Network.Minio.Data.Time import Network.Minio.Data.Time
import Network.Minio.Errors import Network.Minio.Errors
import Text.Printf (printf)
-- these headers are not included in the string to sign when signing a -- these headers are not included in the string to sign when signing a
-- request -- request
ignoredHeaders :: Set.HashSet ByteString ignoredHeaders :: Set.HashSet ByteString
ignoredHeaders = Set.fromList $ map CI.foldedCase ignoredHeaders =
[ H.hAuthorization Set.fromList $
, H.hContentType map
, H.hUserAgent CI.foldedCase
[ H.hAuthorization,
H.hContentType,
H.hUserAgent
] ]
data SignV4Data = SignV4Data { data SignV4Data = SignV4Data
sv4SignTime :: UTCTime { sv4SignTime :: UTCTime,
, sv4Scope :: ByteString sv4Scope :: ByteString,
, sv4CanonicalRequest :: ByteString sv4CanonicalRequest :: ByteString,
, sv4HeadersToSign :: [(ByteString, ByteString)] sv4HeadersToSign :: [(ByteString, ByteString)],
, sv4Output :: [(ByteString, ByteString)] sv4Output :: [(ByteString, ByteString)],
, sv4StringToSign :: ByteString sv4StringToSign :: ByteString,
, sv4SigningKey :: ByteString sv4SigningKey :: ByteString
} deriving (Show) }
deriving (Show)
data SignParams = SignParams { data SignParams = SignParams
spAccessKey :: Text { spAccessKey :: Text,
, spSecretKey :: Text spSecretKey :: Text,
, spTimeStamp :: UTCTime spTimeStamp :: UTCTime,
, spRegion :: Maybe Text spRegion :: Maybe Text,
, spExpirySecs :: Maybe Int spExpirySecs :: Maybe Int,
, spPayloadHash :: Maybe ByteString spPayloadHash :: Maybe ByteString
} deriving (Show) }
deriving (Show)
debugPrintSignV4Data :: SignV4Data -> IO () debugPrintSignV4Data :: SignV4Data -> IO ()
debugPrintSignV4Data (SignV4Data t s cr h2s o sts sk) = do debugPrintSignV4Data (SignV4Data t s cr h2s o sts sk) = do
@ -79,20 +84,21 @@ debugPrintSignV4Data (SignV4Data t s cr h2s o sts sk) = do
B8.putStrLn "END of SignV4Data =========" B8.putStrLn "END of SignV4Data ========="
where where
printBytes b = do printBytes b = do
mapM_ (\x -> B.putStr $ B.concat [show x, " "]) $ B.unpack b mapM_ (\x -> B.putStr $ B.singleton x <> " ") $ B.unpack b
B8.putStrLn "" B8.putStrLn ""
mkAuthHeader :: Text -> ByteString -> ByteString -> ByteString -> H.Header mkAuthHeader :: Text -> ByteString -> ByteString -> ByteString -> H.Header
mkAuthHeader accessKey scope signedHeaderKeys sign = mkAuthHeader accessKey scope signedHeaderKeys sign =
let authValue = B.concat let authValue =
[ "AWS4-HMAC-SHA256 Credential=" B.concat
, toS accessKey [ "AWS4-HMAC-SHA256 Credential=",
, "/" toUtf8 accessKey,
, scope "/",
, ", SignedHeaders=" scope,
, signedHeaderKeys ", SignedHeaders=",
, ", Signature=" signedHeaderKeys,
, sign ", Signature=",
sign
] ]
in (H.hAuthorization, authValue) in (H.hAuthorization, authValue)
@ -110,70 +116,73 @@ mkAuthHeader accessKey scope signedHeaderKeys sign =
-- the request. -- the request.
signV4 :: SignParams -> NC.Request -> [(ByteString, ByteString)] signV4 :: SignParams -> NC.Request -> [(ByteString, ByteString)]
signV4 !sp !req = signV4 !sp !req =
let let region = fromMaybe "" $ spRegion sp
region = fromMaybe "" $ spRegion sp
ts = spTimeStamp sp ts = spTimeStamp sp
scope = mkScope ts region scope = mkScope ts region
accessKey = toS $ spAccessKey sp accessKey = toUtf8 $ spAccessKey sp
secretKey = toS $ spSecretKey sp secretKey = toUtf8 $ spSecretKey sp
expiry = spExpirySecs sp expiry = spExpirySecs sp
sha256Hdr = ("x-amz-content-sha256", sha256Hdr =
fromMaybe "UNSIGNED-PAYLOAD" $ spPayloadHash sp) ( "x-amz-content-sha256",
fromMaybe "UNSIGNED-PAYLOAD" $ spPayloadHash sp
)
-- headers to be added to the request -- headers to be added to the request
datePair = ("X-Amz-Date", awsTimeFormatBS ts) datePair = ("X-Amz-Date", awsTimeFormatBS ts)
computedHeaders = NC.requestHeaders req ++ computedHeaders =
if isJust $ expiry NC.requestHeaders req
++ if isJust $ expiry
then [] then []
else map (\(x, y) -> (mk x, y)) [datePair, sha256Hdr] else map (\(x, y) -> (mk x, y)) [datePair, sha256Hdr]
headersToSign = getHeadersToSign computedHeaders headersToSign = getHeadersToSign computedHeaders
signedHeaderKeys = B.intercalate ";" $ sort $ map fst headersToSign signedHeaderKeys = B.intercalate ";" $ sort $ map fst headersToSign
-- query-parameters to be added before signing for presigned URLs -- query-parameters to be added before signing for presigned URLs
-- (i.e. when `isJust expiry`) -- (i.e. when `isJust expiry`)
authQP = [ ("X-Amz-Algorithm", "AWS4-HMAC-SHA256") authQP =
, ("X-Amz-Credential", B.concat [accessKey, "/", scope]) [ ("X-Amz-Algorithm", "AWS4-HMAC-SHA256"),
, datePair ("X-Amz-Credential", B.concat [accessKey, "/", scope]),
, ("X-Amz-Expires", maybe "" show expiry) datePair,
, ("X-Amz-SignedHeaders", signedHeaderKeys) ("X-Amz-Expires", maybe "" showBS expiry),
("X-Amz-SignedHeaders", signedHeaderKeys)
] ]
finalQP = parseQuery (NC.queryString req) ++ finalQP =
if isJust expiry parseQuery (NC.queryString req)
++ if isJust expiry
then (fmap . fmap) Just authQP then (fmap . fmap) Just authQP
else [] else []
-- 1. compute canonical request -- 1. compute canonical request
canonicalRequest = mkCanonicalRequest False sp canonicalRequest =
mkCanonicalRequest
False
sp
(NC.setQueryString finalQP req) (NC.setQueryString finalQP req)
headersToSign headersToSign
-- 2. compute string to sign -- 2. compute string to sign
stringToSign = mkStringToSign ts scope canonicalRequest stringToSign = mkStringToSign ts scope canonicalRequest
-- 3.1 compute signing key -- 3.1 compute signing key
signingKey = mkSigningKey ts region secretKey signingKey = mkSigningKey ts region secretKey
-- 3.2 compute signature -- 3.2 compute signature
signature = computeSignature stringToSign signingKey signature = computeSignature stringToSign signingKey
-- 4. compute auth header -- 4. compute auth header
authHeader = mkAuthHeader (spAccessKey sp) scope signedHeaderKeys signature authHeader = mkAuthHeader (spAccessKey sp) scope signedHeaderKeys signature
-- finally compute output pairs -- finally compute output pairs
output = if isJust expiry output =
if isJust expiry
then ("X-Amz-Signature", signature) : authQP then ("X-Amz-Signature", signature) : authQP
else [(\(x, y) -> (CI.foldedCase x, y)) authHeader, else
datePair, sha256Hdr] [ (\(x, y) -> (CI.foldedCase x, y)) authHeader,
datePair,
sha256Hdr
]
in output in output
mkScope :: UTCTime -> Text -> ByteString mkScope :: UTCTime -> Text -> ByteString
mkScope ts region = B.intercalate "/" mkScope ts region =
[ toS $ Time.formatTime Time.defaultTimeLocale "%Y%m%d" ts B.intercalate
, toS region "/"
, "s3" [ toUtf8 $ Time.formatTime Time.defaultTimeLocale "%Y%m%d" ts,
, "aws4_request" toUtf8 region,
"s3",
"aws4_request"
] ]
getHeadersToSign :: [Header] -> [(ByteString, ByteString)] getHeadersToSign :: [Header] -> [(ByteString, ByteString)]
@ -181,49 +190,56 @@ getHeadersToSign !h =
filter ((\hdr -> not $ Set.member hdr ignoredHeaders) . fst) $ filter ((\hdr -> not $ Set.member hdr ignoredHeaders) . fst) $
map (\(x, y) -> (CI.foldedCase x, stripBS y)) h map (\(x, y) -> (CI.foldedCase x, stripBS y)) h
mkCanonicalRequest :: Bool -> SignParams -> NC.Request -> [(ByteString, ByteString)] mkCanonicalRequest ::
-> ByteString Bool ->
SignParams ->
NC.Request ->
[(ByteString, ByteString)] ->
ByteString
mkCanonicalRequest !isStreaming !sp !req !headersForSign = mkCanonicalRequest !isStreaming !sp !req !headersForSign =
let let canonicalQueryString =
canonicalQueryString = B.intercalate "&" $ B.intercalate "&"
map (\(x, y) -> B.concat [x, "=", y]) $ $ map (\(x, y) -> B.concat [x, "=", y])
sort $ map (\(x, y) -> $ sort
(uriEncode True x, maybe "" (uriEncode True) y)) $ $ map
(parseQuery $ NC.queryString req) ( \(x, y) ->
(uriEncode True x, maybe "" (uriEncode True) y)
)
$ (parseQuery $ NC.queryString req)
sortedHeaders = sort headersForSign sortedHeaders = sort headersForSign
canonicalHeaders =
canonicalHeaders = B.concat $ B.concat $
map (\(x, y) -> B.concat [x, ":", y, "\n"]) sortedHeaders map (\(x, y) -> B.concat [x, ":", y, "\n"]) sortedHeaders
signedHeaders = B.intercalate ";" $ map fst sortedHeaders signedHeaders = B.intercalate ";" $ map fst sortedHeaders
payloadHashStr = payloadHashStr =
if isStreaming if isStreaming
then "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" then "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
else fromMaybe "UNSIGNED-PAYLOAD" $ spPayloadHash sp else fromMaybe "UNSIGNED-PAYLOAD" $ spPayloadHash sp
in in B.intercalate
B.intercalate "\n" "\n"
[ NC.method req [ NC.method req,
, uriEncode False $ NC.path req uriEncode False $ NC.path req,
, canonicalQueryString canonicalQueryString,
, canonicalHeaders canonicalHeaders,
, signedHeaders signedHeaders,
, payloadHashStr payloadHashStr
] ]
mkStringToSign :: UTCTime -> ByteString -> ByteString -> ByteString mkStringToSign :: UTCTime -> ByteString -> ByteString -> ByteString
mkStringToSign ts !scope !canonicalRequest = B.intercalate "\n" mkStringToSign ts !scope !canonicalRequest =
[ "AWS4-HMAC-SHA256" B.intercalate
, awsTimeFormatBS ts "\n"
, scope [ "AWS4-HMAC-SHA256",
, hashSHA256 canonicalRequest awsTimeFormatBS ts,
scope,
hashSHA256 canonicalRequest
] ]
mkSigningKey :: UTCTime -> Text -> ByteString -> ByteString mkSigningKey :: UTCTime -> Text -> ByteString -> ByteString
mkSigningKey ts region !secretKey = hmacSHA256RawBS "aws4_request" mkSigningKey ts region !secretKey =
hmacSHA256RawBS "aws4_request"
. hmacSHA256RawBS "s3" . hmacSHA256RawBS "s3"
. hmacSHA256RawBS (toS region) . hmacSHA256RawBS (toUtf8 region)
. hmacSHA256RawBS (awsDateFormatBS ts) . hmacSHA256RawBS (awsDateFormatBS ts)
$ B.concat ["AWS4", secretKey] $ B.concat ["AWS4", secretKey]
@ -233,17 +249,18 @@ computeSignature !toSign !key = digestToBase16 $ hmacSHA256 toSign key
-- | Takes a validated Post Policy JSON bytestring, the signing time, -- | Takes a validated Post Policy JSON bytestring, the signing time,
-- and ConnInfo and returns form-data for the POST upload containing -- and ConnInfo and returns form-data for the POST upload containing
-- just the signature and the encoded post-policy. -- just the signature and the encoded post-policy.
signV4PostPolicy :: ByteString -> SignParams signV4PostPolicy ::
-> Map.HashMap Text ByteString ByteString ->
SignParams ->
Map.HashMap Text ByteString
signV4PostPolicy !postPolicyJSON !sp = signV4PostPolicy !postPolicyJSON !sp =
let let stringToSign = Base64.encode postPolicyJSON
stringToSign = Base64.encode postPolicyJSON
region = fromMaybe "" $ spRegion sp region = fromMaybe "" $ spRegion sp
signingKey = mkSigningKey (spTimeStamp sp) region $ toS $ spSecretKey sp signingKey = mkSigningKey (spTimeStamp sp) region $ toUtf8 $ spSecretKey sp
signature = computeSignature stringToSign signingKey signature = computeSignature stringToSign signingKey
in in Map.fromList
Map.fromList [ ("x-amz-signature", signature) [ ("x-amz-signature", signature),
, ("policy", stringToSign) ("policy", stringToSign)
] ]
chunkSizeConstant :: Int chunkSizeConstant :: Int
@ -252,118 +269,114 @@ chunkSizeConstant = 64 * 1024
-- base16Len computes the number of bytes required to represent @n (> 0)@ in -- base16Len computes the number of bytes required to represent @n (> 0)@ in
-- hexadecimal. -- hexadecimal.
base16Len :: Integral a => a -> Int base16Len :: Integral a => a -> Int
base16Len n | n == 0 = 0 base16Len n
| n == 0 = 0
| otherwise = 1 + base16Len (n `div` 16) | otherwise = 1 + base16Len (n `div` 16)
signedStreamLength :: Int64 -> Int64 signedStreamLength :: Int64 -> Int64
signedStreamLength dataLen = signedStreamLength dataLen =
let let chunkSzInt = fromIntegral chunkSizeConstant
chunkSzInt = fromIntegral chunkSizeConstant
(numChunks, lastChunkLen) = quotRem dataLen chunkSzInt (numChunks, lastChunkLen) = quotRem dataLen chunkSzInt
-- Structure of a chunk: -- Structure of a chunk:
-- string(IntHexBase(chunk-size)) + ";chunk-signature=" + signature + \r\n + chunk-data + \r\n -- string(IntHexBase(chunk-size)) + ";chunk-signature=" + signature + \r\n + chunk-data + \r\n
encodedChunkLen csz = fromIntegral (base16Len csz) + 17 + 64 + 2 + csz + 2 encodedChunkLen csz = fromIntegral (base16Len csz) + 17 + 64 + 2 + csz + 2
fullChunkSize = encodedChunkLen chunkSzInt fullChunkSize = encodedChunkLen chunkSzInt
lastChunkSize = bool 0 (encodedChunkLen lastChunkLen) $ lastChunkLen > 0 lastChunkSize = bool 0 (encodedChunkLen lastChunkLen) $ lastChunkLen > 0
finalChunkSize = 1 + 17 + 64 + 2 + 2 finalChunkSize = 1 + 17 + 64 + 2 + 2
in in numChunks * fullChunkSize + lastChunkSize + finalChunkSize
numChunks * fullChunkSize + lastChunkSize + finalChunkSize
signV4Stream :: Int64 -> SignParams -> NC.Request signV4Stream ::
-> (C.ConduitT () ByteString (C.ResourceT IO) () -> NC.Request) Int64 ->
-- -> ([Header], C.ConduitT () ByteString (C.ResourceT IO) () -> NC.RequestBody) SignParams ->
NC.Request ->
(C.ConduitT () ByteString (C.ResourceT IO) () -> NC.Request)
-- -> ([Header], C.ConduitT () ByteString (C.ResourceT IO) () -> NC.RequestBody)
signV4Stream !payloadLength !sp !req = signV4Stream !payloadLength !sp !req =
let let ts = spTimeStamp sp
ts = spTimeStamp sp
addContentEncoding hs = addContentEncoding hs =
let ceMay = headMay $ filter (\(x, _) -> x == "content-encoding") hs let ceMay = headMay $ filter (\(x, _) -> x == "content-encoding") hs
in case ceMay of in case ceMay of
Nothing -> ("content-encoding", "aws-chunked") : hs Nothing -> ("content-encoding", "aws-chunked") : hs
Just (_, ce) -> ("content-encoding", ce <> ",aws-chunked") : Just (_, ce) ->
filter (\(x, _) -> x /= "content-encoding") hs ("content-encoding", ce <> ",aws-chunked")
: filter (\(x, _) -> x /= "content-encoding") hs
-- headers to be added to the request -- headers to be added to the request
datePair = ("X-Amz-Date", awsTimeFormatBS ts) datePair = ("X-Amz-Date", awsTimeFormatBS ts)
computedHeaders = addContentEncoding $ computedHeaders =
addContentEncoding $
datePair : NC.requestHeaders req datePair : NC.requestHeaders req
-- headers specific to streaming signature -- headers specific to streaming signature
signedContentLength = signedStreamLength payloadLength signedContentLength = signedStreamLength payloadLength
streamingHeaders :: [Header] streamingHeaders :: [Header]
streamingHeaders = streamingHeaders =
[ ("x-amz-decoded-content-length", show payloadLength) [ ("x-amz-decoded-content-length", showBS payloadLength),
, ("content-length", show signedContentLength ) ("content-length", showBS signedContentLength),
, ("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD") ("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")
] ]
headersToSign = getHeadersToSign $ computedHeaders ++ streamingHeaders headersToSign = getHeadersToSign $ computedHeaders ++ streamingHeaders
signedHeaderKeys = B.intercalate ";" $ sort $ map fst headersToSign signedHeaderKeys = B.intercalate ";" $ sort $ map fst headersToSign
finalQP = parseQuery (NC.queryString req) finalQP = parseQuery (NC.queryString req)
-- 1. Compute Seed Signature -- 1. Compute Seed Signature
-- 1.1 Canonical Request -- 1.1 Canonical Request
canonicalReq = mkCanonicalRequest True sp canonicalReq =
mkCanonicalRequest
True
sp
(NC.setQueryString finalQP req) (NC.setQueryString finalQP req)
headersToSign headersToSign
region = fromMaybe "" $ spRegion sp region = fromMaybe "" $ spRegion sp
scope = mkScope ts region scope = mkScope ts region
accessKey = spAccessKey sp accessKey = spAccessKey sp
secretKey = spSecretKey sp secretKey = spSecretKey sp
-- 1.2 String toSign -- 1.2 String toSign
stringToSign = mkStringToSign ts scope canonicalReq stringToSign = mkStringToSign ts scope canonicalReq
-- 1.3 Compute signature -- 1.3 Compute signature
-- 1.3.1 compute signing key -- 1.3.1 compute signing key
signingKey = mkSigningKey ts region $ toS secretKey signingKey = mkSigningKey ts region $ toUtf8 secretKey
-- 1.3.2 Compute signature -- 1.3.2 Compute signature
seedSignature = computeSignature stringToSign signingKey seedSignature = computeSignature stringToSign signingKey
-- 1.3.3 Compute Auth Header -- 1.3.3 Compute Auth Header
authHeader = mkAuthHeader accessKey scope signedHeaderKeys seedSignature authHeader = mkAuthHeader accessKey scope signedHeaderKeys seedSignature
-- 1.4 Updated headers for the request -- 1.4 Updated headers for the request
finalReqHeaders = authHeader : (computedHeaders ++ streamingHeaders) finalReqHeaders = authHeader : (computedHeaders ++ streamingHeaders)
-- headersToAdd = authHeader : datePair : streamingHeaders -- headersToAdd = authHeader : datePair : streamingHeaders
toHexStr n = B8.pack $ printf "%x" n toHexStr n = B8.pack $ printf "%x" n
(numParts, lastPSize) = payloadLength `quotRem` fromIntegral chunkSizeConstant (numParts, lastPSize) = payloadLength `quotRem` fromIntegral chunkSizeConstant
-- Function to compute string to sign for each chunk. -- Function to compute string to sign for each chunk.
chunkStrToSign prevSign currChunkHash = chunkStrToSign prevSign currChunkHash =
B.intercalate "\n" B.intercalate
[ "AWS4-HMAC-SHA256-PAYLOAD" "\n"
, awsTimeFormatBS ts [ "AWS4-HMAC-SHA256-PAYLOAD",
, scope awsTimeFormatBS ts,
, prevSign scope,
, hashSHA256 "" prevSign,
, currChunkHash hashSHA256 "",
currChunkHash
] ]
-- Read n byte from upstream and return a strict bytestring. -- Read n byte from upstream and return a strict bytestring.
mustTakeN n = do mustTakeN n = do
bs <- toS <$> (C.takeCE n C..| C.sinkLazy) bs <- LB.toStrict <$> (C.takeCE n C..| C.sinkLazy)
when (B.length bs /= n) $ when (B.length bs /= n) $
throwIO MErrVStreamingBodyUnexpectedEOF throwIO MErrVStreamingBodyUnexpectedEOF
return bs return bs
signerConduit n lps prevSign = signerConduit n lps prevSign =
-- First case encodes a full chunk of length -- First case encodes a full chunk of length
-- 'chunkSizeConstant'. -- 'chunkSizeConstant'.
if | n > 0 -> do if
| n > 0 -> do
bs <- mustTakeN chunkSizeConstant bs <- mustTakeN chunkSizeConstant
let strToSign = chunkStrToSign prevSign (hashSHA256 bs) let strToSign = chunkStrToSign prevSign (hashSHA256 bs)
nextSign = computeSignature strToSign signingKey nextSign = computeSignature strToSign signingKey
chunkBS = toHexStr chunkSizeConstant chunkBS =
toHexStr chunkSizeConstant
<> ";chunk-signature=" <> ";chunk-signature="
<> nextSign <> "\r\n" <> bs <> "\r\n" <> nextSign
<> "\r\n"
<> bs
<> "\r\n"
C.yield chunkBS C.yield chunkBS
signerConduit (n-1) lps nextSign signerConduit (n -1) lps nextSign
-- Second case encodes the last chunk which is smaller than -- Second case encodes the last chunk which is smaller than
-- 'chunkSizeConstant' -- 'chunkSizeConstant'
@ -371,8 +384,12 @@ signV4Stream !payloadLength !sp !req =
bs <- mustTakeN $ fromIntegral lps bs <- mustTakeN $ fromIntegral lps
let strToSign = chunkStrToSign prevSign (hashSHA256 bs) let strToSign = chunkStrToSign prevSign (hashSHA256 bs)
nextSign = computeSignature strToSign signingKey nextSign = computeSignature strToSign signingKey
chunkBS = toHexStr lps <> ";chunk-signature=" chunkBS =
<> nextSign <> "\r\n" <> bs <> "\r\n" toHexStr lps <> ";chunk-signature="
<> nextSign
<> "\r\n"
<> bs
<> "\r\n"
C.yield chunkBS C.yield chunkBS
signerConduit 0 0 nextSign signerConduit 0 0 nextSign
@ -383,9 +400,10 @@ signV4Stream !payloadLength !sp !req =
nextSign = computeSignature strToSign signingKey nextSign = computeSignature strToSign signingKey
lastChunkBS = "0;chunk-signature=" <> nextSign <> "\r\n\r\n" lastChunkBS = "0;chunk-signature=" <> nextSign <> "\r\n\r\n"
C.yield lastChunkBS C.yield lastChunkBS
in in \src ->
\src -> req { NC.requestHeaders = finalReqHeaders req
, NC.requestBody = { NC.requestHeaders = finalReqHeaders,
NC.requestBody =
NC.requestBodySource signedContentLength $ NC.requestBodySource signedContentLength $
src C..| signerConduit numParts lastPSize seedSignature src C..| signerConduit numParts lastPSize seedSignature
} }

View File

@ -27,26 +27,29 @@ import qualified Data.HashMap.Strict as H
import qualified Data.List as List import qualified Data.List as List
import qualified Data.Text as T import qualified Data.Text as T
import Data.Text.Read (decimal) import Data.Text.Read (decimal)
import Data.Time (defaultTimeLocale, parseTimeM, import Data.Time
rfc822DateFormat) ( defaultTimeLocale,
parseTimeM,
rfc822DateFormat,
)
import Lib.Prelude
import Network.HTTP.Conduit (Response) import Network.HTTP.Conduit (Response)
import qualified Network.HTTP.Conduit as NC import qualified Network.HTTP.Conduit as NC
import qualified Network.HTTP.Types as HT import qualified Network.HTTP.Types as HT
import qualified Network.HTTP.Types.Header as Hdr import qualified Network.HTTP.Types.Header as Hdr
import Network.Minio.Data
import Network.Minio.Data.ByteString
import Network.Minio.JsonParser (parseErrResponseJSON)
import Network.Minio.XmlParser (parseErrResponse)
import qualified System.IO as IO import qualified System.IO as IO
import qualified UnliftIO as U import qualified UnliftIO as U
import qualified UnliftIO.Async as A import qualified UnliftIO.Async as A
import qualified UnliftIO.MVar as UM import qualified UnliftIO.MVar as UM
import Lib.Prelude allocateReadFile ::
(MonadUnliftIO m, R.MonadResource m) =>
import Network.Minio.Data FilePath ->
import Network.Minio.Data.ByteString m (R.ReleaseKey, Handle)
import Network.Minio.JsonParser (parseErrResponseJSON)
import Network.Minio.XmlParser (parseErrResponse)
allocateReadFile :: (MonadUnliftIO m, R.MonadResource m)
=> FilePath -> m (R.ReleaseKey, Handle)
allocateReadFile fp = do allocateReadFile fp = do
(rk, hdlE) <- R.allocate (openReadFile fp) cleanup (rk, hdlE) <- R.allocate (openReadFile fp) cleanup
either (\(e :: IOException) -> throwIO e) (return . (rk,)) hdlE either (\(e :: IOException) -> throwIO e) (return . (rk,)) hdlE
@ -56,8 +59,10 @@ allocateReadFile fp = do
-- | Queries the file size from the handle. Catches any file operation -- | Queries the file size from the handle. Catches any file operation
-- exceptions and returns Nothing instead. -- exceptions and returns Nothing instead.
getFileSize :: (MonadUnliftIO m, R.MonadResource m) getFileSize ::
=> Handle -> m (Maybe Int64) (MonadUnliftIO m, R.MonadResource m) =>
Handle ->
m (Maybe Int64)
getFileSize h = do getFileSize h = do
resE <- liftIO $ try $ fromIntegral <$> IO.hFileSize h resE <- liftIO $ try $ fromIntegral <$> IO.hFileSize h
case resE of case resE of
@ -66,8 +71,10 @@ getFileSize h = do
-- | Queries if handle is seekable. Catches any file operation -- | Queries if handle is seekable. Catches any file operation
-- exceptions and return False instead. -- exceptions and return False instead.
isHandleSeekable :: (R.MonadResource m, MonadUnliftIO m) isHandleSeekable ::
=> Handle -> m Bool (R.MonadResource m, MonadUnliftIO m) =>
Handle ->
m Bool
isHandleSeekable h = do isHandleSeekable h = do
resE <- liftIO $ try $ IO.hIsSeekable h resE <- liftIO $ try $ IO.hIsSeekable h
case resE of case resE of
@ -78,8 +85,11 @@ isHandleSeekable h = do
-- the given action on it. Exceptions of type MError are caught and -- the given action on it. Exceptions of type MError are caught and
-- returned - both during file handle allocation and when the action -- returned - both during file handle allocation and when the action
-- is run. -- is run.
withNewHandle :: (MonadUnliftIO m, R.MonadResource m) withNewHandle ::
=> FilePath -> (Handle -> m a) -> m (Either IOException a) (MonadUnliftIO m, R.MonadResource m) =>
FilePath ->
(Handle -> m a) ->
m (Either IOException a)
withNewHandle fp fileAction = do withNewHandle fp fileAction = do
-- opening a handle can throw MError exception. -- opening a handle can throw MError exception.
handleE <- try $ allocateReadFile fp handleE <- try $ allocateReadFile fp
@ -107,11 +117,13 @@ getMetadata =
toMaybeMetadataHeader :: (Text, Text) -> Maybe (Text, Text) toMaybeMetadataHeader :: (Text, Text) -> Maybe (Text, Text)
toMaybeMetadataHeader (k, v) = toMaybeMetadataHeader (k, v) =
(, v) <$> userMetadataHeaderNameMaybe k (,v) <$> userMetadataHeaderNameMaybe k
getNonUserMetadataMap :: [(Text, Text)] -> H.HashMap Text Text getNonUserMetadataMap :: [(Text, Text)] -> H.HashMap Text Text
getNonUserMetadataMap = H.fromList getNonUserMetadataMap =
. filter ( isNothing H.fromList
. filter
( isNothing
. userMetadataHeaderNameMaybe . userMetadataHeaderNameMaybe
. fst . fst
) )
@ -119,7 +131,8 @@ getNonUserMetadataMap = H.fromList
-- | This function collects all headers starting with `x-amz-meta-` -- | This function collects all headers starting with `x-amz-meta-`
-- and strips off this prefix, and returns a map. -- and strips off this prefix, and returns a map.
getUserMetadataMap :: [(Text, Text)] -> H.HashMap Text Text getUserMetadataMap :: [(Text, Text)] -> H.HashMap Text Text
getUserMetadataMap = H.fromList getUserMetadataMap =
H.fromList
. mapMaybe toMaybeMetadataHeader . mapMaybe toMaybeMetadataHeader
getLastModifiedHeader :: [HT.Header] -> Maybe UTCTime getLastModifiedHeader :: [HT.Header] -> Maybe UTCTime
@ -132,17 +145,19 @@ getContentLength hs = do
nbs <- decodeUtf8Lenient <$> lookupHeader Hdr.hContentLength hs nbs <- decodeUtf8Lenient <$> lookupHeader Hdr.hContentLength hs
fst <$> hush (decimal nbs) fst <$> hush (decimal nbs)
decodeUtf8Lenient :: ByteString -> Text decodeUtf8Lenient :: ByteString -> Text
decodeUtf8Lenient = decodeUtf8With lenientDecode decodeUtf8Lenient = decodeUtf8With lenientDecode
isSuccessStatus :: HT.Status -> Bool isSuccessStatus :: HT.Status -> Bool
isSuccessStatus sts = let s = HT.statusCode sts isSuccessStatus sts =
let s = HT.statusCode sts
in (s >= 200 && s < 300) in (s >= 200 && s < 300)
httpLbs :: MonadIO m httpLbs ::
=> NC.Request -> NC.Manager MonadIO m =>
-> m (NC.Response LByteString) NC.Request ->
NC.Manager ->
m (NC.Response LByteString)
httpLbs req mgr = do httpLbs req mgr = do
respE <- liftIO $ tryHttpEx $ NC.httpLbs req mgr respE <- liftIO $ tryHttpEx $ NC.httpLbs req mgr
resp <- either throwIO return respE resp <- either throwIO return respE
@ -154,21 +169,25 @@ httpLbs req mgr = do
Just "application/json" -> do Just "application/json" -> do
sErr <- parseErrResponseJSON $ NC.responseBody resp sErr <- parseErrResponseJSON $ NC.responseBody resp
throwIO sErr throwIO sErr
_ ->
_ -> throwIO $ NC.HttpExceptionRequest req $ throwIO $ NC.HttpExceptionRequest req $
NC.StatusCodeException (void resp) (show resp) NC.StatusCodeException (void resp) (showBS resp)
return resp return resp
where where
tryHttpEx :: IO (NC.Response LByteString) tryHttpEx ::
-> IO (Either NC.HttpException (NC.Response LByteString)) IO (NC.Response LByteString) ->
IO (Either NC.HttpException (NC.Response LByteString))
tryHttpEx = try tryHttpEx = try
contentTypeMay resp = lookupHeader Hdr.hContentType $ contentTypeMay resp =
lookupHeader Hdr.hContentType $
NC.responseHeaders resp NC.responseHeaders resp
http :: (MonadUnliftIO m, R.MonadResource m) http ::
=> NC.Request -> NC.Manager (MonadUnliftIO m, R.MonadResource m) =>
-> m (Response (C.ConduitT () ByteString m ())) NC.Request ->
NC.Manager ->
m (Response (C.ConduitT () ByteString m ()))
http req mgr = do http req mgr = do
respE <- tryHttpEx $ NC.http req mgr respE <- tryHttpEx $ NC.http req mgr
resp <- either throwIO return respE resp <- either throwIO return respE
@ -178,25 +197,30 @@ http req mgr = do
respBody <- C.connect (NC.responseBody resp) CB.sinkLbs respBody <- C.connect (NC.responseBody resp) CB.sinkLbs
sErr <- parseErrResponse respBody sErr <- parseErrResponse respBody
throwIO sErr throwIO sErr
_ -> do _ -> do
content <- LB.toStrict . NC.responseBody <$> NC.lbsResponse resp content <- LB.toStrict . NC.responseBody <$> NC.lbsResponse resp
throwIO $ NC.HttpExceptionRequest req $ throwIO $ NC.HttpExceptionRequest req $
NC.StatusCodeException (void resp) content NC.StatusCodeException (void resp) content
return resp return resp
where where
tryHttpEx :: (MonadUnliftIO m) => m a tryHttpEx ::
-> m (Either NC.HttpException a) (MonadUnliftIO m) =>
m a ->
m (Either NC.HttpException a)
tryHttpEx = try tryHttpEx = try
contentTypeMay resp = lookupHeader Hdr.hContentType $ contentTypeMay resp =
lookupHeader Hdr.hContentType $
NC.responseHeaders resp NC.responseHeaders resp
-- Similar to mapConcurrently but limits the number of threads that -- Similar to mapConcurrently but limits the number of threads that
-- can run using a quantity semaphore. -- can run using a quantity semaphore.
limitedMapConcurrently :: MonadUnliftIO m limitedMapConcurrently ::
=> Int -> (t -> m a) -> [t] -> m [a] MonadUnliftIO m =>
Int ->
(t -> m a) ->
[t] ->
m [a]
limitedMapConcurrently 0 _ _ = return [] limitedMapConcurrently 0 _ _ = return []
limitedMapConcurrently count act args = do limitedMapConcurrently count act args = do
t' <- U.newTVarIO count t' <- U.newTVarIO count
@ -205,17 +229,15 @@ limitedMapConcurrently count act args = do
where where
wThread t arg = wThread t arg =
U.bracket_ (waitSem t) (signalSem t) $ act arg U.bracket_ (waitSem t) (signalSem t) $ act arg
-- quantity semaphore implementation using TVar -- quantity semaphore implementation using TVar
waitSem t = U.atomically $ do waitSem t = U.atomically $ do
v <- U.readTVar t v <- U.readTVar t
if v > 0 if v > 0
then U.writeTVar t (v-1) then U.writeTVar t (v -1)
else U.retrySTM else U.retrySTM
signalSem t = U.atomically $ do signalSem t = U.atomically $ do
v <- U.readTVar t v <- U.readTVar t
U.writeTVar t (v+1) U.writeTVar t (v + 1)
-- helper function to 'drop' empty optional parameter. -- helper function to 'drop' empty optional parameter.
mkQuery :: Text -> Maybe Text -> Maybe (Text, Text) mkQuery :: Text -> Maybe Text -> Maybe (Text, Text)
@ -232,23 +254,30 @@ mkOptionalParams params = HT.toQuery $ uncurry mkQuery <$> params
-- received. Does not throw any errors. -- received. Does not throw any errors.
chunkBSConduit :: (Monad m) => [Int] -> C.ConduitM ByteString ByteString m () chunkBSConduit :: (Monad m) => [Int] -> C.ConduitM ByteString ByteString m ()
chunkBSConduit [] = return () chunkBSConduit [] = return ()
chunkBSConduit (s:ss) = do chunkBSConduit (s : ss) = do
bs <- fmap LB.toStrict $ C.takeCE s C..| C.sinkLazy bs <- fmap LB.toStrict $ C.takeCE s C..| C.sinkLazy
if | B.length bs == s -> C.yield bs >> chunkBSConduit ss if
| B.length bs == s -> C.yield bs >> chunkBSConduit ss
| B.length bs > 0 -> C.yield bs | B.length bs > 0 -> C.yield bs
| otherwise -> return () | otherwise -> return ()
-- | Select part sizes - the logic is that the minimum part-size will -- | Select part sizes - the logic is that the minimum part-size will
-- be 64MiB. -- be 64MiB.
selectPartSizes :: Int64 -> [(PartNumber, Int64, Int64)] selectPartSizes :: Int64 -> [(PartNumber, Int64, Int64)]
selectPartSizes size = uncurry (List.zip3 [1..]) $ selectPartSizes size =
List.unzip $ loop 0 size uncurry (List.zip3 [1 ..])
$ List.unzip
$ loop 0 size
where where
ceil :: Double -> Int64 ceil :: Double -> Int64
ceil = ceiling ceil = ceiling
partSize = max minPartSize (ceil $ fromIntegral size / partSize =
fromIntegral maxMultipartParts) max
minPartSize
( ceil $
fromIntegral size
/ fromIntegral maxMultipartParts
)
m = fromIntegral partSize m = fromIntegral partSize
loop st sz loop st sz
| st > sz = [] | st > sz = []

View File

@ -15,31 +15,32 @@
-- --
module Network.Minio.XmlGenerator module Network.Minio.XmlGenerator
( mkCreateBucketConfig ( mkCreateBucketConfig,
, mkCompleteMultipartUploadRequest mkCompleteMultipartUploadRequest,
, mkPutNotificationRequest mkPutNotificationRequest,
, mkSelectRequest mkSelectRequest,
) where )
where
import qualified Data.ByteString.Lazy as LBS import qualified Data.ByteString.Lazy as LBS
import qualified Data.HashMap.Strict as H import qualified Data.HashMap.Strict as H
import qualified Data.Text as T import qualified Data.Text as T
import Text.XML
import Lib.Prelude import Lib.Prelude
import Network.Minio.Data import Network.Minio.Data
import Text.XML
-- | Create a bucketConfig request body XML -- | Create a bucketConfig request body XML
mkCreateBucketConfig :: Text -> Region -> ByteString mkCreateBucketConfig :: Text -> Region -> ByteString
mkCreateBucketConfig ns location = LBS.toStrict $ renderLBS def bucketConfig mkCreateBucketConfig ns location = LBS.toStrict $ renderLBS def bucketConfig
where where
s3Element n = Element (s3Name ns n) mempty s3Element n = Element (s3Name ns n) mempty
root = s3Element "CreateBucketConfiguration" root =
[ NodeElement $ s3Element "LocationConstraint" s3Element
[ NodeContent location] "CreateBucketConfiguration"
[ NodeElement $
s3Element
"LocationConstraint"
[NodeContent location]
] ]
bucketConfig = Document (Prologue [] Nothing []) root [] bucketConfig = Document (Prologue [] Nothing []) root []
@ -48,29 +49,45 @@ mkCompleteMultipartUploadRequest :: [PartTuple] -> ByteString
mkCompleteMultipartUploadRequest partInfo = mkCompleteMultipartUploadRequest partInfo =
LBS.toStrict $ renderLBS def cmur LBS.toStrict $ renderLBS def cmur
where where
root = Element "CompleteMultipartUpload" mempty $ root =
Element "CompleteMultipartUpload" mempty $
map (NodeElement . mkPart) partInfo map (NodeElement . mkPart) partInfo
mkPart (n, etag) = Element "Part" mempty mkPart (n, etag) =
[ NodeElement $ Element "PartNumber" mempty Element
[NodeContent $ T.pack $ show n] "Part"
, NodeElement $ Element "ETag" mempty mempty
[ NodeElement $
Element
"PartNumber"
mempty
[NodeContent $ T.pack $ show n],
NodeElement $
Element
"ETag"
mempty
[NodeContent etag] [NodeContent etag]
] ]
cmur = Document (Prologue [] Nothing []) root [] cmur = Document (Prologue [] Nothing []) root []
-- Simplified XML representation without element attributes. -- Simplified XML representation without element attributes.
data XNode = XNode Text [XNode] data XNode
= XNode Text [XNode]
| XLeaf Text Text | XLeaf Text Text
deriving (Eq, Show) deriving (Eq, Show)
toXML :: Text -> XNode -> ByteString toXML :: Text -> XNode -> ByteString
toXML ns node = LBS.toStrict $ renderLBS def $ toXML ns node =
LBS.toStrict $ renderLBS def $
Document (Prologue [] Nothing []) (xmlNode node) [] Document (Prologue [] Nothing []) (xmlNode node) []
where where
xmlNode :: XNode -> Element xmlNode :: XNode -> Element
xmlNode (XNode name nodes) = Element (s3Name ns name) mempty $ xmlNode (XNode name nodes) =
Element (s3Name ns name) mempty $
map (NodeElement . xmlNode) nodes map (NodeElement . xmlNode) nodes
xmlNode (XLeaf name content) = Element (s3Name ns name) mempty xmlNode (XLeaf name content) =
Element
(s3Name ns name)
mempty
[NodeContent content] [NodeContent content]
class ToXNode a where class ToXNode a where
@ -80,23 +97,28 @@ instance ToXNode Event where
toXNode = XLeaf "Event" . show toXNode = XLeaf "Event" . show
instance ToXNode Notification where instance ToXNode Notification where
toXNode (Notification qc tc lc) = XNode "NotificationConfiguration" $ toXNode (Notification qc tc lc) =
map (toXNodesWithArnName "QueueConfiguration" "Queue") qc ++ XNode "NotificationConfiguration" $
map (toXNodesWithArnName "TopicConfiguration" "Topic") tc ++ map (toXNodesWithArnName "QueueConfiguration" "Queue") qc
map (toXNodesWithArnName "CloudFunctionConfiguration" "CloudFunction") lc ++ map (toXNodesWithArnName "TopicConfiguration" "Topic") tc
++ map (toXNodesWithArnName "CloudFunctionConfiguration" "CloudFunction") lc
toXNodesWithArnName :: Text -> Text -> NotificationConfig -> XNode toXNodesWithArnName :: Text -> Text -> NotificationConfig -> XNode
toXNodesWithArnName eltName arnName (NotificationConfig id arn events fRule) = toXNodesWithArnName eltName arnName (NotificationConfig id arn events fRule) =
XNode eltName $ [XLeaf "Id" id, XLeaf arnName arn] ++ map toXNode events ++ XNode eltName $
[toXNode fRule] [XLeaf "Id" id, XLeaf arnName arn] ++ map toXNode events
++ [toXNode fRule]
instance ToXNode Filter where instance ToXNode Filter where
toXNode (Filter (FilterKey (FilterRules rules))) = toXNode (Filter (FilterKey (FilterRules rules))) =
XNode "Filter" [XNode "S3Key" (map getFRXNode rules)] XNode "Filter" [XNode "S3Key" (map getFRXNode rules)]
getFRXNode :: FilterRule -> XNode getFRXNode :: FilterRule -> XNode
getFRXNode (FilterRule n v) = XNode "FilterRule" [ XLeaf "Name" n getFRXNode (FilterRule n v) =
, XLeaf "Value" v XNode
"FilterRule"
[ XLeaf "Name" n,
XLeaf "Value" v
] ]
mkPutNotificationRequest :: Text -> Notification -> ByteString mkPutNotificationRequest :: Text -> Notification -> ByteString
@ -106,60 +128,103 @@ mkSelectRequest :: SelectRequest -> ByteString
mkSelectRequest r = LBS.toStrict $ renderLBS def sr mkSelectRequest r = LBS.toStrict $ renderLBS def sr
where where
sr = Document (Prologue [] Nothing []) root [] sr = Document (Prologue [] Nothing []) root []
root = Element "SelectRequest" mempty $ root =
[ NodeElement (Element "Expression" mempty Element "SelectRequest" mempty $
[NodeContent $ srExpression r]) [ NodeElement
, NodeElement (Element "ExpressionType" mempty ( Element
[NodeContent $ show $ srExpressionType r]) "Expression"
, NodeElement (Element "InputSerialization" mempty $ mempty
inputSerializationNodes $ srInputSerialization r) [NodeContent $ srExpression r]
, NodeElement (Element "OutputSerialization" mempty $ ),
outputSerializationNodes $ srOutputSerialization r) NodeElement
] ++ maybe [] reqProgElem (srRequestProgressEnabled r) ( Element
reqProgElem enabled = [NodeElement "ExpressionType"
(Element "RequestProgress" mempty mempty
[NodeElement [NodeContent $ show $ srExpressionType r]
(Element "Enabled" mempty ),
[NodeContent NodeElement
(if enabled then "TRUE" else "FALSE")] ( Element "InputSerialization" mempty
$ inputSerializationNodes
$ srInputSerialization r
),
NodeElement
( Element "OutputSerialization" mempty
$ outputSerializationNodes
$ srOutputSerialization r
)
]
++ maybe [] reqProgElem (srRequestProgressEnabled r)
reqProgElem enabled =
[ NodeElement
( Element
"RequestProgress"
mempty
[ NodeElement
( Element
"Enabled"
mempty
[ NodeContent
(if enabled then "TRUE" else "FALSE")
]
) )
] ]
) )
] ]
inputSerializationNodes is = comprTypeNode (isCompressionType is) ++ inputSerializationNodes is =
[NodeElement $ formatNode (isFormatInfo is)] comprTypeNode (isCompressionType is)
comprTypeNode (Just c) = [NodeElement $ Element "CompressionType" mempty ++ [NodeElement $ formatNode (isFormatInfo is)]
[NodeContent $ case c of comprTypeNode (Just c) =
[ NodeElement $
Element
"CompressionType"
mempty
[ NodeContent $ case c of
CompressionTypeNone -> "NONE" CompressionTypeNone -> "NONE"
CompressionTypeGzip -> "GZIP" CompressionTypeGzip -> "GZIP"
CompressionTypeBzip2 -> "BZIP2" CompressionTypeBzip2 -> "BZIP2"
] ]
] ]
comprTypeNode Nothing = [] comprTypeNode Nothing = []
kvElement (k, v) = Element (Name k Nothing Nothing) mempty [NodeContent v] kvElement (k, v) = Element (Name k Nothing Nothing) mempty [NodeContent v]
formatNode (InputFormatCSV (CSVProp h)) = formatNode (InputFormatCSV (CSVProp h)) =
Element "CSV" mempty Element
"CSV"
mempty
(map NodeElement $ map kvElement $ H.toList h) (map NodeElement $ map kvElement $ H.toList h)
formatNode (InputFormatJSON p) = formatNode (InputFormatJSON p) =
Element "JSON" mempty Element
[NodeElement "JSON"
(Element "Type" mempty mempty
[NodeContent $ case jsonipType p of [ NodeElement
( Element
"Type"
mempty
[ NodeContent $ case jsonipType p of
JSONTypeDocument -> "DOCUMENT" JSONTypeDocument -> "DOCUMENT"
JSONTypeLines -> "LINES" JSONTypeLines -> "LINES"
] ]
) )
] ]
formatNode InputFormatParquet = Element "Parquet" mempty [] formatNode InputFormatParquet = Element "Parquet" mempty []
outputSerializationNodes (OutputSerializationJSON j) = outputSerializationNodes (OutputSerializationJSON j) =
[NodeElement (Element "JSON" mempty $ [ NodeElement
rdElem $ jsonopRecordDelimiter j)] ( Element "JSON" mempty
$ rdElem
$ jsonopRecordDelimiter j
)
]
outputSerializationNodes (OutputSerializationCSV (CSVProp h)) = outputSerializationNodes (OutputSerializationCSV (CSVProp h)) =
[NodeElement $ Element "CSV" mempty [ NodeElement $
(map NodeElement $ map kvElement $ H.toList h)] Element
"CSV"
mempty
(map NodeElement $ map kvElement $ H.toList h)
]
rdElem Nothing = [] rdElem Nothing = []
rdElem (Just t) = [NodeElement $ Element "RecordDelimiter" mempty rdElem (Just t) =
[NodeContent t]] [ NodeElement $
Element
"RecordDelimiter"
mempty
[NodeContent t]
]

View File

@ -15,19 +15,20 @@
-- --
module Network.Minio.XmlParser module Network.Minio.XmlParser
( parseListBuckets ( parseListBuckets,
, parseLocation parseLocation,
, parseNewMultipartUpload parseNewMultipartUpload,
, parseCompleteMultipartUploadResponse parseCompleteMultipartUploadResponse,
, parseCopyObjectResponse parseCopyObjectResponse,
, parseListObjectsResponse parseListObjectsResponse,
, parseListObjectsV1Response parseListObjectsV1Response,
, parseListUploadsResponse parseListUploadsResponse,
, parseListPartsResponse parseListPartsResponse,
, parseErrResponse parseErrResponse,
, parseNotification parseNotification,
, parseSelectProgress parseSelectProgress,
) where )
where
import qualified Data.ByteString.Lazy as LB import qualified Data.ByteString.Lazy as LB
import qualified Data.HashMap.Strict as H import qualified Data.HashMap.Strict as H
@ -35,14 +36,11 @@ import Data.List (zip3, zip4, zip6)
import qualified Data.Text as T import qualified Data.Text as T
import Data.Text.Read (decimal) import Data.Text.Read (decimal)
import Data.Time import Data.Time
import Text.XML
import Text.XML.Cursor hiding (bool)
import Lib.Prelude import Lib.Prelude
import Network.Minio.Data import Network.Minio.Data
import Network.Minio.Errors import Network.Minio.Errors
import Text.XML
import Text.XML.Cursor hiding (bool)
-- | Represent the time format string returned by S3 API calls. -- | Represent the time format string returned by S3 API calls.
s3TimeFormat :: [Char] s3TimeFormat :: [Char]
@ -58,11 +56,13 @@ uncurry6 f (a, b, c, d, e, g) = f a b c d e g
-- | Parse time strings from XML -- | Parse time strings from XML
parseS3XMLTime :: MonadIO m => Text -> m UTCTime parseS3XMLTime :: MonadIO m => Text -> m UTCTime
parseS3XMLTime t = parseS3XMLTime t =
maybe (throwIO $ MErrVXmlParse $ "timestamp parse failure: " <> t) return $ maybe (throwIO $ MErrVXmlParse $ "timestamp parse failure: " <> t) return
parseTimeM True defaultTimeLocale s3TimeFormat $ T.unpack t $ parseTimeM True defaultTimeLocale s3TimeFormat
$ T.unpack t
parseDecimal :: (MonadIO m, Integral a) => Text -> m a parseDecimal :: (MonadIO m, Integral a) => Text -> m a
parseDecimal numStr = either (throwIO . MErrVXmlParse . show) return $ parseDecimal numStr =
either (throwIO . MErrVXmlParse . show) return $
fst <$> decimal numStr fst <$> decimal numStr
parseDecimals :: (MonadIO m, Integral a) => [Text] -> m [a] parseDecimals :: (MonadIO m, Integral a) => [Text] -> m [a]
@ -72,7 +72,8 @@ s3Elem :: Text -> Text -> Axis
s3Elem ns = element . s3Name ns s3Elem ns = element . s3Name ns
parseRoot :: (MonadIO m) => LByteString -> m Cursor parseRoot :: (MonadIO m) => LByteString -> m Cursor
parseRoot = either (throwIO . MErrVXmlParse . show) (return . fromDocument) parseRoot =
either (throwIO . MErrVXmlParse . show) (return . fromDocument)
. parseLBS def . parseLBS def
-- | Parse the response XML of a list buckets call. -- | Parse the response XML of a list buckets call.
@ -80,8 +81,7 @@ parseListBuckets :: (MonadReader env m, HasSvcNamespace env, MonadIO m) => LByte
parseListBuckets xmldata = do parseListBuckets xmldata = do
r <- parseRoot xmldata r <- parseRoot xmldata
ns <- asks getSvcNamespace ns <- asks getSvcNamespace
let let s3Elem' = s3Elem ns
s3Elem' = s3Elem ns
names = r $// s3Elem' "Bucket" &// s3Elem' "Name" &/ content names = r $// s3Elem' "Bucket" &// s3Elem' "Name" &/ content
timeStrings = r $// s3Elem' "Bucket" &// s3Elem' "CreationDate" &/ content timeStrings = r $// s3Elem' "Bucket" &// s3Elem' "CreationDate" &/ content
@ -116,27 +116,24 @@ parseCopyObjectResponse :: (MonadReader env m, HasSvcNamespace env, MonadIO m) =
parseCopyObjectResponse xmldata = do parseCopyObjectResponse xmldata = do
r <- parseRoot xmldata r <- parseRoot xmldata
ns <- asks getSvcNamespace ns <- asks getSvcNamespace
let let s3Elem' = s3Elem ns
s3Elem' = s3Elem ns
mtimeStr = T.concat $ r $// s3Elem' "LastModified" &/ content mtimeStr = T.concat $ r $// s3Elem' "LastModified" &/ content
mtime <- parseS3XMLTime mtimeStr mtime <- parseS3XMLTime mtimeStr
return (T.concat $ r $// s3Elem' "ETag" &/ content, mtime) return (T.concat $ r $// s3Elem' "ETag" &/ content, mtime)
-- | Parse the response XML of a list objects v1 call. -- | Parse the response XML of a list objects v1 call.
parseListObjectsV1Response :: (MonadReader env m, HasSvcNamespace env, MonadIO m) parseListObjectsV1Response ::
=> LByteString -> m ListObjectsV1Result (MonadReader env m, HasSvcNamespace env, MonadIO m) =>
LByteString ->
m ListObjectsV1Result
parseListObjectsV1Response xmldata = do parseListObjectsV1Response xmldata = do
r <- parseRoot xmldata r <- parseRoot xmldata
ns <- asks getSvcNamespace ns <- asks getSvcNamespace
let let s3Elem' = s3Elem ns
s3Elem' = s3Elem ns
hasMore = ["true"] == (r $/ s3Elem' "IsTruncated" &/ content) hasMore = ["true"] == (r $/ s3Elem' "IsTruncated" &/ content)
nextMarker = headMay $ r $/ s3Elem' "NextMarker" &/ content nextMarker = headMay $ r $/ s3Elem' "NextMarker" &/ content
prefixes = r $/ s3Elem' "CommonPrefixes" &/ s3Elem' "Prefix" &/ content prefixes = r $/ s3Elem' "CommonPrefixes" &/ s3Elem' "Prefix" &/ content
keys = r $/ s3Elem' "Contents" &/ s3Elem' "Key" &/ content keys = r $/ s3Elem' "Contents" &/ s3Elem' "Key" &/ content
modTimeStr = r $/ s3Elem' "Contents" &/ s3Elem' "LastModified" &/ content modTimeStr = r $/ s3Elem' "Contents" &/ s3Elem' "LastModified" &/ content
etagsList = r $/ s3Elem' "Contents" &/ s3Elem' "ETag" &/ content etagsList = r $/ s3Elem' "Contents" &/ s3Elem' "ETag" &/ content
@ -148,8 +145,8 @@ parseListObjectsV1Response xmldata = do
modTimes <- mapM parseS3XMLTime modTimeStr modTimes <- mapM parseS3XMLTime modTimeStr
sizes <- parseDecimals sizeStr sizes <- parseDecimals sizeStr
let let objects =
objects = map (uncurry6 ObjectInfo) $ map (uncurry6 ObjectInfo) $
zip6 keys modTimes etags sizes (repeat H.empty) (repeat H.empty) zip6 keys modTimes etags sizes (repeat H.empty) (repeat H.empty)
return $ ListObjectsV1Result hasMore nextMarker objects prefixes return $ ListObjectsV1Result hasMore nextMarker objects prefixes
@ -159,14 +156,10 @@ parseListObjectsResponse :: (MonadReader env m, HasSvcNamespace env, MonadIO m)
parseListObjectsResponse xmldata = do parseListObjectsResponse xmldata = do
r <- parseRoot xmldata r <- parseRoot xmldata
ns <- asks getSvcNamespace ns <- asks getSvcNamespace
let let s3Elem' = s3Elem ns
s3Elem' = s3Elem ns
hasMore = ["true"] == (r $/ s3Elem' "IsTruncated" &/ content) hasMore = ["true"] == (r $/ s3Elem' "IsTruncated" &/ content)
nextToken = headMay $ r $/ s3Elem' "NextContinuationToken" &/ content nextToken = headMay $ r $/ s3Elem' "NextContinuationToken" &/ content
prefixes = r $/ s3Elem' "CommonPrefixes" &/ s3Elem' "Prefix" &/ content prefixes = r $/ s3Elem' "CommonPrefixes" &/ s3Elem' "Prefix" &/ content
keys = r $/ s3Elem' "Contents" &/ s3Elem' "Key" &/ content keys = r $/ s3Elem' "Contents" &/ s3Elem' "Key" &/ content
modTimeStr = r $/ s3Elem' "Contents" &/ s3Elem' "LastModified" &/ content modTimeStr = r $/ s3Elem' "Contents" &/ s3Elem' "LastModified" &/ content
etagsList = r $/ s3Elem' "Contents" &/ s3Elem' "ETag" &/ content etagsList = r $/ s3Elem' "Contents" &/ s3Elem' "ETag" &/ content
@ -178,8 +171,8 @@ parseListObjectsResponse xmldata = do
modTimes <- mapM parseS3XMLTime modTimeStr modTimes <- mapM parseS3XMLTime modTimeStr
sizes <- parseDecimals sizeStr sizes <- parseDecimals sizeStr
let let objects =
objects = map (uncurry6 ObjectInfo) $ map (uncurry6 ObjectInfo) $
zip6 keys modTimes etags sizes (repeat H.empty) (repeat H.empty) zip6 keys modTimes etags sizes (repeat H.empty) (repeat H.empty)
return $ ListObjectsResult hasMore nextToken objects prefixes return $ ListObjectsResult hasMore nextToken objects prefixes
@ -189,8 +182,7 @@ parseListUploadsResponse :: (MonadReader env m, HasSvcNamespace env, MonadIO m)
parseListUploadsResponse xmldata = do parseListUploadsResponse xmldata = do
r <- parseRoot xmldata r <- parseRoot xmldata
ns <- asks getSvcNamespace ns <- asks getSvcNamespace
let let s3Elem' = s3Elem ns
s3Elem' = s3Elem ns
hasMore = ["true"] == (r $/ s3Elem' "IsTruncated" &/ content) hasMore = ["true"] == (r $/ s3Elem' "IsTruncated" &/ content)
prefixes = r $/ s3Elem' "CommonPrefixes" &/ s3Elem' "Prefix" &/ content prefixes = r $/ s3Elem' "CommonPrefixes" &/ s3Elem' "Prefix" &/ content
nextKey = headMay $ r $/ s3Elem' "NextKeyMarker" &/ content nextKey = headMay $ r $/ s3Elem' "NextKeyMarker" &/ content
@ -201,8 +193,7 @@ parseListUploadsResponse xmldata = do
uploadInitTimes <- mapM parseS3XMLTime uploadInitTimeStr uploadInitTimes <- mapM parseS3XMLTime uploadInitTimeStr
let let uploads = zip3 uploadKeys uploadIds uploadInitTimes
uploads = zip3 uploadKeys uploadIds uploadInitTimes
return $ ListUploadsResult hasMore nextKey nextUpload uploads prefixes return $ ListUploadsResult hasMore nextKey nextUpload uploads prefixes
@ -210,8 +201,7 @@ parseListPartsResponse :: (MonadReader env m, HasSvcNamespace env, MonadIO m) =>
parseListPartsResponse xmldata = do parseListPartsResponse xmldata = do
r <- parseRoot xmldata r <- parseRoot xmldata
ns <- asks getSvcNamespace ns <- asks getSvcNamespace
let let s3Elem' = s3Elem ns
s3Elem' = s3Elem ns
hasMore = ["true"] == (r $/ s3Elem' "IsTruncated" &/ content) hasMore = ["true"] == (r $/ s3Elem' "IsTruncated" &/ content)
nextPartNumStr = headMay $ r $/ s3Elem' "NextPartNumberMarker" &/ content nextPartNumStr = headMay $ r $/ s3Elem' "NextPartNumberMarker" &/ content
partNumberStr = r $/ s3Elem' "Part" &/ s3Elem' "PartNumber" &/ content partNumberStr = r $/ s3Elem' "Part" &/ s3Elem' "PartNumber" &/ content
@ -224,13 +214,12 @@ parseListPartsResponse xmldata = do
partNumbers <- parseDecimals partNumberStr partNumbers <- parseDecimals partNumberStr
nextPartNum <- parseDecimals $ maybeToList nextPartNumStr nextPartNum <- parseDecimals $ maybeToList nextPartNumStr
let let partInfos =
partInfos = map (uncurry4 ObjectPartInfo) $ map (uncurry4 ObjectPartInfo) $
zip4 partNumbers partETags partSizes partModTimes zip4 partNumbers partETags partSizes partModTimes
return $ ListPartsResult hasMore (listToMaybe nextPartNum) partInfos return $ ListPartsResult hasMore (listToMaybe nextPartNum) partInfos
parseErrResponse :: (MonadIO m) => LByteString -> m ServiceErr parseErrResponse :: (MonadIO m) => LByteString -> m ServiceErr
parseErrResponse xmldata = do parseErrResponse xmldata = do
r <- parseRoot xmldata r <- parseRoot xmldata
@ -250,28 +239,31 @@ parseNotification xmldata = do
<*> (mapM (parseNode ns "Topic") tcfg) <*> (mapM (parseNode ns "Topic") tcfg)
<*> (mapM (parseNode ns "CloudFunction") lcfg) <*> (mapM (parseNode ns "CloudFunction") lcfg)
where where
getFilterRule ns c = getFilterRule ns c =
let name = T.concat $ c $/ s3Elem ns "Name" &/ content let name = T.concat $ c $/ s3Elem ns "Name" &/ content
value = T.concat $ c $/ s3Elem ns "Value" &/ content value = T.concat $ c $/ s3Elem ns "Value" &/ content
in FilterRule name value in FilterRule name value
parseNode ns arnName nodeData = do parseNode ns arnName nodeData = do
let c = fromNode nodeData let c = fromNode nodeData
id = T.concat $ c $/ s3Elem ns "Id" &/ content id = T.concat $ c $/ s3Elem ns "Id" &/ content
arn = T.concat $ c $/ s3Elem ns arnName &/ content arn = T.concat $ c $/ s3Elem ns arnName &/ content
events = catMaybes $ map textToEvent $ c $/ s3Elem ns "Event" &/ content events = catMaybes $ map textToEvent $ c $/ s3Elem ns "Event" &/ content
rules = c $/ s3Elem ns "Filter" &/ s3Elem ns "S3Key" &/ rules =
s3Elem ns "FilterRule" &| getFilterRule ns c $/ s3Elem ns "Filter" &/ s3Elem ns "S3Key"
return $ NotificationConfig id arn events &/ s3Elem ns "FilterRule" &| getFilterRule ns
return $
NotificationConfig
id
arn
events
(Filter $ FilterKey $ FilterRules rules) (Filter $ FilterKey $ FilterRules rules)
parseSelectProgress :: MonadIO m => ByteString -> m Progress parseSelectProgress :: MonadIO m => ByteString -> m Progress
parseSelectProgress xmldata = do parseSelectProgress xmldata = do
r <- parseRoot $ LB.fromStrict xmldata r <- parseRoot $ LB.fromStrict xmldata
let bScanned = T.concat $ r $/ element "BytesScanned" &/ content let bScanned = T.concat $ r $/ element "BytesScanned" &/ content
bProcessed = T.concat $ r $/element "BytesProcessed" &/ content bProcessed = T.concat $ r $/ element "BytesProcessed" &/ content
bReturned = T.concat $ r $/element "BytesReturned" &/ content bReturned = T.concat $ r $/ element "BytesReturned" &/ content
Progress <$> parseDecimal bScanned Progress <$> parseDecimal bScanned
<*> parseDecimal bProcessed <*> parseDecimal bProcessed
<*> parseDecimal bReturned <*> parseDecimal bReturned

View File

@ -15,7 +15,7 @@
# resolver: # resolver:
# name: custom-snapshot # name: custom-snapshot
# location: "./custom-snapshot.yaml" # location: "./custom-snapshot.yaml"
resolver: lts-14.6 resolver: lts-16.0
# User packages to be built. # User packages to be built.
# Various formats can be used as shown in the example below. # Various formats can be used as shown in the example below.
@ -39,7 +39,9 @@ packages:
- '.' - '.'
# Dependency packages to be pulled from upstream that are not in the resolver # Dependency packages to be pulled from upstream that are not in the resolver
# (e.g., acme-missiles-0.3) # (e.g., acme-missiles-0.3)
extra-deps: [] extra-deps:
- unliftio-core-0.2.0.1
- protolude-0.3.0
# Override default flag values for local packages and extra-deps # Override default flag values for local packages and extra-deps
flags: {} flags: {}

View File

@ -3,10 +3,24 @@
# For more information, please see the documentation at: # For more information, please see the documentation at:
# https://docs.haskellstack.org/en/stable/lock_files # https://docs.haskellstack.org/en/stable/lock_files
packages: [] packages:
- completed:
hackage: unliftio-core-0.2.0.1@sha256:9b3e44ea9aacacbfc35b3b54015af450091916ac3618a41868ebf6546977659a,1082
pantry-tree:
size: 328
sha256: e81c5a1e82ec2cd68cbbbec9cd60567363abe02257fa1370a906f6754b6818b8
original:
hackage: unliftio-core-0.2.0.1
- completed:
hackage: protolude-0.3.0@sha256:8361b811b420585b122a7ba715aa5923834db6e8c36309bf267df2dbf66b95ef,2693
pantry-tree:
size: 1644
sha256: babf32b414f25f790b7a4ce6bae5c960bc51a11a289e7c47335b222e6762560c
original:
hackage: protolude-0.3.0
snapshots: snapshots:
- completed: - completed:
size: 524127 size: 531237
url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/14/6.yaml url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/16/0.yaml
sha256: dc70dfb45e2c32f54719819bd055f46855dd4b3bd2e58b9f3f38729a2d553fbb sha256: 210e15b7043e2783115afe16b0d54914b1611cdaa73f3ca3ca7f8e0847ff54e5
original: lts-14.6 original: lts-16.0

View File

@ -1,4 +1,5 @@
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
-- --
-- MinIO Haskell SDK, (C) 2017-2019 MinIO, Inc. -- MinIO Haskell SDK, (C) 2017-2019 MinIO, Inc.
-- --
@ -15,11 +16,6 @@
-- limitations under the License. -- limitations under the License.
-- --
import qualified Test.QuickCheck as Q
import Test.Tasty
import Test.Tasty.HUnit
import Test.Tasty.QuickCheck as QC
import Conduit (replicateC) import Conduit (replicateC)
import qualified Conduit as C import qualified Conduit as C
import qualified Control.Monad.Trans.Resource as R import qualified Control.Monad.Trans.Resource as R
@ -31,21 +27,23 @@ import qualified Data.HashMap.Strict as H
import qualified Data.Text as T import qualified Data.Text as T
import Data.Time (fromGregorian) import Data.Time (fromGregorian)
import qualified Data.Time as Time import qualified Data.Time as Time
import Lib.Prelude
import qualified Network.HTTP.Client.MultipartFormData as Form import qualified Network.HTTP.Client.MultipartFormData as Form
import qualified Network.HTTP.Conduit as NC import qualified Network.HTTP.Conduit as NC
import qualified Network.HTTP.Types as HT import qualified Network.HTTP.Types as HT
import System.Directory (getTemporaryDirectory)
import System.Environment (lookupEnv)
import qualified System.IO as SIO
import Lib.Prelude
import Network.Minio import Network.Minio
import Network.Minio.Data import Network.Minio.Data
import Network.Minio.Data.Crypto import Network.Minio.Data.Crypto
import Network.Minio.PutObject import Network.Minio.PutObject
import Network.Minio.S3API import Network.Minio.S3API
import Network.Minio.Utils import Network.Minio.Utils
import System.Directory (getTemporaryDirectory)
import System.Environment (lookupEnv)
import qualified System.IO as SIO
import qualified Test.QuickCheck as Q
import Test.Tasty
import Test.Tasty.HUnit
import Test.Tasty.QuickCheck as QC
main :: IO () main :: IO ()
main = defaultMain tests main = defaultMain tests
@ -57,16 +55,19 @@ tests = testGroup "Tests" [liveServerUnitTests]
randomDataSrc :: MonadIO m => Int64 -> C.ConduitM () ByteString m () randomDataSrc :: MonadIO m => Int64 -> C.ConduitM () ByteString m ()
randomDataSrc s' = genBS s' randomDataSrc s' = genBS s'
where where
concatIt bs n = BS.concat $ replicate (fromIntegral q) bs ++ concatIt bs n =
[BS.take (fromIntegral r) bs] BS.concat $
where (q, r) = n `divMod` fromIntegral (BS.length bs) replicate (fromIntegral q) bs
++ [BS.take (fromIntegral r) bs]
where
(q, r) = n `divMod` fromIntegral (BS.length bs)
genBS s = do genBS s = do
w8s <- liftIO $ generate $ Q.vectorOf 64 (Q.choose (0, 255)) w8s <- liftIO $ generate $ Q.vectorOf 64 (Q.choose (0, 255))
let byteArr64 = BS.pack w8s let byteArr64 = BS.pack w8s
if s < oneMiB if s < oneMiB
then yield $ concatIt byteArr64 s then yield $ concatIt byteArr64 s
else do yield $ concatIt byteArr64 oneMiB else do
yield $ concatIt byteArr64 oneMiB
genBS (s - oneMiB) genBS (s - oneMiB)
mkRandFile :: R.MonadResource m => Int64 -> m FilePath mkRandFile :: R.MonadResource m => Int64 -> m FilePath
@ -86,8 +87,10 @@ loadTestServer = do
(Just _, Nothing) -> setCreds (Credentials "minio" "minio123") "http://localhost:9000" (Just _, Nothing) -> setCreds (Credentials "minio" "minio123") "http://localhost:9000"
(Nothing, _) -> minioPlayCI (Nothing, _) -> minioPlayCI
funTestWithBucket :: TestName funTestWithBucket ::
-> (([Char] -> Minio ()) -> Bucket -> Minio ()) -> TestTree TestName ->
(([Char] -> Minio ()) -> Bucket -> Minio ()) ->
TestTree
funTestWithBucket t minioTest = testCaseSteps t $ \step -> do funTestWithBucket t minioTest = testCaseSteps t $ \step -> do
-- generate a random name for the bucket -- generate a random name for the bucket
bktSuffix <- liftIO $ generate $ Q.vectorOf 10 (Q.choose ('a', 'z')) bktSuffix <- liftIO $ generate $ Q.vectorOf 10 (Q.choose ('a', 'z'))
@ -104,24 +107,26 @@ funTestWithBucket t minioTest = testCaseSteps t $ \step -> do
isRight ret @? ("Functional test " ++ t ++ " failed => " ++ show ret) isRight ret @? ("Functional test " ++ t ++ " failed => " ++ show ret)
liveServerUnitTests :: TestTree liveServerUnitTests :: TestTree
liveServerUnitTests = testGroup "Unit tests against a live server" liveServerUnitTests =
[ basicTests testGroup
, listingTest "Unit tests against a live server"
, highLevelListingTest [ basicTests,
, lowLevelMultipartTest listingTest,
, putObjectSizeTest highLevelListingTest,
, putObjectNoSizeTest lowLevelMultipartTest,
, multipartTest putObjectSizeTest,
, putObjectContentTypeTest putObjectNoSizeTest,
, putObjectContentLanguageTest multipartTest,
, putObjectStorageClassTest putObjectContentTypeTest,
, putObjectUserMetadataTest putObjectContentLanguageTest,
, getObjectTest putObjectStorageClassTest,
, copyObjectTests putObjectUserMetadataTest,
, presignedUrlFunTest getObjectTest,
, presignedPostPolicyFunTest copyObjectTests,
, bucketPolicyFunTest presignedUrlFunTest,
, getNPutSSECTest presignedPostPolicyFunTest,
bucketPolicyFunTest,
getNPutSSECTest
] ]
basicTests :: TestTree basicTests :: TestTree
@ -129,10 +134,12 @@ basicTests = funTestWithBucket "Basic tests" $
\step bucket -> do \step bucket -> do
step "getService works and contains the test bucket." step "getService works and contains the test bucket."
buckets <- getService buckets <- getService
unless (length (filter (== bucket) $ map biName buckets) == 1) $ unless (length (filter (== bucket) $ map biName buckets) == 1)
liftIO $ $ liftIO
assertFailure ("The bucket " ++ show bucket ++ $ assertFailure
" was expected to exist.") ( "The bucket " ++ show bucket
++ " was expected to exist."
)
step "makeBucket again to check if BucketAlreadyOwnedByYou exception is raised." step "makeBucket again to check if BucketAlreadyOwnedByYou exception is raised."
mbE <- try $ makeBucket bucket Nothing mbE <- try $ makeBucket bucket Nothing
@ -165,32 +172,54 @@ basicTests = funTestWithBucket "Basic tests" $
let unmodifiedTime = UTCTime (fromGregorian 2010 11 26) 69857 let unmodifiedTime = UTCTime (fromGregorian 2010 11 26) 69857
step "fGetObject an object which is modified now but requesting as un-modified in past, check for exception" step "fGetObject an object which is modified now but requesting as un-modified in past, check for exception"
resE <- try $ fGetObject bucket "lsb-release" outFile defaultGetObjectOptions { resE <-
gooIfUnmodifiedSince = (Just unmodifiedTime) try $
fGetObject
bucket
"lsb-release"
outFile
defaultGetObjectOptions
{ gooIfUnmodifiedSince = (Just unmodifiedTime)
} }
case resE of case resE of
Left exn -> liftIO $ exn @?= ServiceErr "PreconditionFailed" "At least one of the pre-conditions you specified did not hold" Left exn -> liftIO $ exn @?= ServiceErr "PreconditionFailed" "At least one of the pre-conditions you specified did not hold"
_ -> return () _ -> return ()
step "fGetObject an object with no matching etag, check for exception" step "fGetObject an object with no matching etag, check for exception"
resE1 <- try $ fGetObject bucket "lsb-release" outFile defaultGetObjectOptions { resE1 <-
gooIfMatch = (Just "invalid-etag") try $
fGetObject
bucket
"lsb-release"
outFile
defaultGetObjectOptions
{ gooIfMatch = (Just "invalid-etag")
} }
case resE1 of case resE1 of
Left exn -> liftIO $ exn @?= ServiceErr "PreconditionFailed" "At least one of the pre-conditions you specified did not hold" Left exn -> liftIO $ exn @?= ServiceErr "PreconditionFailed" "At least one of the pre-conditions you specified did not hold"
_ -> return () _ -> return ()
step "fGetObject an object with no valid range, check for exception" step "fGetObject an object with no valid range, check for exception"
resE2 <- try $ fGetObject bucket "lsb-release" outFile defaultGetObjectOptions { resE2 <-
gooRange = (Just $ HT.ByteRangeFromTo 100 200) try $
fGetObject
bucket
"lsb-release"
outFile
defaultGetObjectOptions
{ gooRange = (Just $ HT.ByteRangeFromTo 100 200)
} }
case resE2 of case resE2 of
Left exn -> liftIO $ exn @?= ServiceErr "InvalidRange" "The requested range is not satisfiable" Left exn -> liftIO $ exn @?= ServiceErr "InvalidRange" "The requested range is not satisfiable"
_ -> return () _ -> return ()
step "fGetObject on object with a valid range" step "fGetObject on object with a valid range"
fGetObject bucket "lsb-release" outFile defaultGetObjectOptions { fGetObject
gooRange = (Just $ HT.ByteRangeFrom 1) bucket
"lsb-release"
outFile
defaultGetObjectOptions
{ gooRange = (Just $ HT.ByteRangeFrom 1)
} }
step "fGetObject a non-existent object and check for NoSuchKey exception" step "fGetObject a non-existent object and check for NoSuchKey exception"
@ -247,8 +276,9 @@ lowLevelMultipartTest = funTestWithBucket "Low-level Multipart Test" $
step "Retrieve the created object and check size" step "Retrieve the created object and check size"
fGetObject bucket object destFile defaultGetObjectOptions fGetObject bucket object destFile defaultGetObjectOptions
gotSize <- withNewHandle destFile getFileSize gotSize <- withNewHandle destFile getFileSize
liftIO $ gotSize == Right (Just mb15) @? liftIO $
"Wrong file size of put file after getting" gotSize == Right (Just mb15)
@? "Wrong file size of put file after getting"
step "Cleanup actions" step "Cleanup actions"
removeObject bucket object removeObject bucket object
@ -270,8 +300,9 @@ putObjectSizeTest = funTestWithBucket "PutObject of conduit source with size" $
destFile <- mkRandFile 0 destFile <- mkRandFile 0
fGetObject bucket obj destFile defaultGetObjectOptions fGetObject bucket obj destFile defaultGetObjectOptions
gotSize <- withNewHandle destFile getFileSize gotSize <- withNewHandle destFile getFileSize
liftIO $ gotSize == Right (Just mb1) @? liftIO $
"Wrong file size of put file after getting" gotSize == Right (Just mb1)
@? "Wrong file size of put file after getting"
step "Cleanup actions" step "Cleanup actions"
deleteObject bucket obj deleteObject bucket obj
@ -293,8 +324,9 @@ putObjectNoSizeTest = funTestWithBucket "PutObject of conduit source with no siz
destFile <- mkRandFile 0 destFile <- mkRandFile 0
fGetObject bucket obj destFile defaultGetObjectOptions fGetObject bucket obj destFile defaultGetObjectOptions
gotSize <- withNewHandle destFile getFileSize gotSize <- withNewHandle destFile getFileSize
liftIO $ gotSize == Right (Just mb70) @? liftIO $
"Wrong file size of put file after getting" gotSize == Right (Just mb70)
@? "Wrong file size of put file after getting"
step "Cleanup actions" step "Cleanup actions"
deleteObject bucket obj deleteObject bucket obj
@ -306,14 +338,20 @@ highLevelListingTest = funTestWithBucket "High-level listObjects Test" $
step "put 3 objects" step "put 3 objects"
let expectedObjects = ["dir/o1", "dir/dir1/o2", "dir/dir2/o3", "o4"] let expectedObjects = ["dir/o1", "dir/dir1/o2", "dir/dir2/o3", "o4"]
extractObjectsFromList os = extractObjectsFromList os =
mapM (\t -> case t of mapM
( \t -> case t of
ListItemObject o -> Just $ oiObject o ListItemObject o -> Just $ oiObject o
_ -> Nothing) os _ -> Nothing
)
os
expectedNonRecList = ["o4", "dir/"] expectedNonRecList = ["o4", "dir/"]
extractObjectsAndDirsFromList os = extractObjectsAndDirsFromList os =
map (\t -> case t of map
( \t -> case t of
ListItemObject o -> oiObject o ListItemObject o -> oiObject o
ListItemPrefix d -> d) os ListItemPrefix d -> d
)
os
forM_ expectedObjects $ forM_ expectedObjects $
\obj -> fPutObject bucket obj "/etc/lsb-release" defaultPutObjectOptions \obj -> fPutObject bucket obj "/etc/lsb-release" defaultPutObjectOptions
@ -326,9 +364,11 @@ highLevelListingTest = funTestWithBucket "High-level listObjects Test" $
step "High-level recursive listing of objects" step "High-level recursive listing of objects"
objects <- C.runConduit $ listObjects bucket Nothing True C..| sinkList objects <- C.runConduit $ listObjects bucket Nothing True C..| sinkList
liftIO $ assertEqual "Objects match failed!" liftIO
(Just $ sort expectedObjects) $ $ assertEqual
extractObjectsFromList objects "Objects match failed!"
(Just $ sort expectedObjects)
$ extractObjectsFromList objects
step "High-level listing of objects (version 1)" step "High-level listing of objects (version 1)"
itemsV1 <- C.runConduit $ listObjectsV1 bucket Nothing False C..| sinkList itemsV1 <- C.runConduit $ listObjectsV1 bucket Nothing False C..| sinkList
@ -336,34 +376,50 @@ highLevelListingTest = funTestWithBucket "High-level listObjects Test" $
extractObjectsAndDirsFromList itemsV1 extractObjectsAndDirsFromList itemsV1
step "High-level recursive listing of objects (version 1)" step "High-level recursive listing of objects (version 1)"
objectsV1 <- C.runConduit $ listObjectsV1 bucket Nothing True C..| objectsV1 <-
sinkList C.runConduit $
listObjectsV1 bucket Nothing True
C..| sinkList
liftIO $ assertEqual "Objects match failed!" liftIO
(Just $ sort expectedObjects) $ $ assertEqual
extractObjectsFromList objectsV1 "Objects match failed!"
(Just $ sort expectedObjects)
$ extractObjectsFromList objectsV1
let expectedPrefListing = ["dir/o1", "dir/dir1/", "dir/dir2/"] let expectedPrefListing = ["dir/o1", "dir/dir1/", "dir/dir2/"]
expectedPrefListingRec = Just ["dir/dir1/o2", "dir/dir2/o3", "dir/o1"] expectedPrefListingRec = Just ["dir/dir1/o2", "dir/dir2/o3", "dir/o1"]
step "High-level listing with prefix" step "High-level listing with prefix"
prefItems <- C.runConduit $ listObjects bucket (Just "dir/") False C..| sinkList prefItems <- C.runConduit $ listObjects bucket (Just "dir/") False C..| sinkList
liftIO $ assertEqual "Objects/Dirs under prefix match failed!" liftIO
expectedPrefListing $ extractObjectsAndDirsFromList prefItems $ assertEqual
"Objects/Dirs under prefix match failed!"
expectedPrefListing
$ extractObjectsAndDirsFromList prefItems
step "High-level listing with prefix recursive" step "High-level listing with prefix recursive"
prefItemsRec <- C.runConduit $ listObjects bucket (Just "dir/") True C..| sinkList prefItemsRec <- C.runConduit $ listObjects bucket (Just "dir/") True C..| sinkList
liftIO $ assertEqual "Objects/Dirs under prefix match recursive failed!" liftIO
expectedPrefListingRec $ extractObjectsFromList prefItemsRec $ assertEqual
"Objects/Dirs under prefix match recursive failed!"
expectedPrefListingRec
$ extractObjectsFromList prefItemsRec
step "High-level listing with prefix (version 1)" step "High-level listing with prefix (version 1)"
prefItemsV1 <- C.runConduit $ listObjectsV1 bucket (Just "dir/") False C..| sinkList prefItemsV1 <- C.runConduit $ listObjectsV1 bucket (Just "dir/") False C..| sinkList
liftIO $ assertEqual "Objects/Dirs under prefix match failed!" liftIO
expectedPrefListing $ extractObjectsAndDirsFromList prefItemsV1 $ assertEqual
"Objects/Dirs under prefix match failed!"
expectedPrefListing
$ extractObjectsAndDirsFromList prefItemsV1
step "High-level listing with prefix recursive (version 1)" step "High-level listing with prefix recursive (version 1)"
prefItemsRecV1 <- C.runConduit $ listObjectsV1 bucket (Just "dir/") True C..| sinkList prefItemsRecV1 <- C.runConduit $ listObjectsV1 bucket (Just "dir/") True C..| sinkList
liftIO $ assertEqual "Objects/Dirs under prefix match recursive failed!" liftIO
expectedPrefListingRec $ extractObjectsFromList prefItemsRecV1 $ assertEqual
"Objects/Dirs under prefix match recursive failed!"
expectedPrefListingRec
$ extractObjectsFromList prefItemsRecV1
step "Cleanup actions" step "Cleanup actions"
forM_ expectedObjects $ forM_ expectedObjects $
@ -372,14 +428,15 @@ highLevelListingTest = funTestWithBucket "High-level listObjects Test" $
step "High-level listIncompleteUploads Test" step "High-level listIncompleteUploads Test"
let object = "newmpupload" let object = "newmpupload"
step "create 10 multipart uploads" step "create 10 multipart uploads"
forM_ [1..10::Int] $ \_ -> do forM_ [1 .. 10 :: Int] $ \_ -> do
uid <- newMultipartUpload bucket object [] uid <- newMultipartUpload bucket object []
liftIO $ (T.length uid > 0) @? ("Got an empty multipartUpload Id.") liftIO $ (T.length uid > 0) @? ("Got an empty multipartUpload Id.")
step "High-level listing of incomplete multipart uploads" step "High-level listing of incomplete multipart uploads"
uploads <- C.runConduit $ uploads <-
listIncompleteUploads bucket (Just "newmpupload") True C..| C.runConduit $
sinkList listIncompleteUploads bucket (Just "newmpupload") True
C..| sinkList
liftIO $ length uploads @?= 10 liftIO $ length uploads @?= 10
step "cleanup" step "cleanup"
@ -396,11 +453,13 @@ highLevelListingTest = funTestWithBucket "High-level listObjects Test" $
step "put object parts 1..10" step "put object parts 1..10"
inputFile <- mkRandFile mb5 inputFile <- mkRandFile mb5
h <- liftIO $ SIO.openBinaryFile inputFile SIO.ReadMode h <- liftIO $ SIO.openBinaryFile inputFile SIO.ReadMode
forM_ [1..10] $ \pnum -> forM_ [1 .. 10] $ \pnum ->
putObjectPart bucket object uid pnum [] $ PayloadH h 0 mb5 putObjectPart bucket object uid pnum [] $ PayloadH h 0 mb5
step "fetch list parts" step "fetch list parts"
incompleteParts <- C.runConduit $ listIncompleteParts bucket object uid incompleteParts <-
C.runConduit $
listIncompleteParts bucket object uid
C..| sinkList C..| sinkList
liftIO $ length incompleteParts @?= 10 liftIO $ length incompleteParts @?= 10
@ -411,25 +470,36 @@ listingTest :: TestTree
listingTest = funTestWithBucket "Listing Test" $ \step bucket -> do listingTest = funTestWithBucket "Listing Test" $ \step bucket -> do
step "listObjects' test" step "listObjects' test"
step "put 10 objects" step "put 10 objects"
let objects = (\s ->T.concat ["lsb-release", T.pack (show s)]) <$> [1..10::Int] let objects = (\s -> T.concat ["lsb-release", T.pack (show s)]) <$> [1 .. 10 :: Int]
forM_ [1..10::Int] $ \s -> forM_ [1 .. 10 :: Int] $ \s ->
fPutObject bucket (T.concat ["lsb-release", T.pack (show s)]) "/etc/lsb-release" defaultPutObjectOptions fPutObject bucket (T.concat ["lsb-release", T.pack (show s)]) "/etc/lsb-release" defaultPutObjectOptions
step "Simple list" step "Simple list"
res <- listObjects' bucket Nothing Nothing Nothing Nothing res <- listObjects' bucket Nothing Nothing Nothing Nothing
let expectedObjects = sort objects let expectedObjects = sort objects
liftIO $ assertEqual "Objects match failed!" expectedObjects liftIO $
assertEqual
"Objects match failed!"
expectedObjects
(map oiObject $ lorObjects res) (map oiObject $ lorObjects res)
step "Simple list version 1" step "Simple list version 1"
resV1 <- listObjectsV1' bucket Nothing Nothing Nothing Nothing resV1 <- listObjectsV1' bucket Nothing Nothing Nothing Nothing
let expected = sort $ map (T.concat . let expected =
("lsb-release":) . sort $
(\x -> [x]) . map
T.pack . ( T.concat
show) [1..10::Int] . ("lsb-release" :)
liftIO $ assertEqual "Objects match failed!" expected . (\x -> [x])
. T.pack
. show
)
[1 .. 10 :: Int]
liftIO $
assertEqual
"Objects match failed!"
expected
(map oiObject $ lorObjects' resV1) (map oiObject $ lorObjects' resV1)
step "Cleanup actions" step "Cleanup actions"
@ -438,13 +508,19 @@ listingTest = funTestWithBucket "Listing Test" $ \step bucket -> do
step "listIncompleteUploads' test" step "listIncompleteUploads' test"
step "create 10 multipart uploads" step "create 10 multipart uploads"
let object = "newmpupload" let object = "newmpupload"
forM_ [1..10::Int] $ \_ -> do forM_ [1 .. 10 :: Int] $ \_ -> do
uid <- newMultipartUpload bucket object [] uid <- newMultipartUpload bucket object []
liftIO $ (T.length uid > 0) @? ("Got an empty multipartUpload Id.") liftIO $ (T.length uid > 0) @? ("Got an empty multipartUpload Id.")
step "list incomplete multipart uploads" step "list incomplete multipart uploads"
incompleteUploads <- listIncompleteUploads' bucket (Just "newmpupload") Nothing incompleteUploads <-
Nothing Nothing Nothing listIncompleteUploads'
bucket
(Just "newmpupload")
Nothing
Nothing
Nothing
Nothing
liftIO $ (length $ lurUploads incompleteUploads) @?= 10 liftIO $ (length $ lurUploads incompleteUploads) @?= 10
step "cleanup" step "cleanup"
@ -461,7 +537,7 @@ listingTest = funTestWithBucket "Listing Test" $ \step bucket -> do
step "put object parts 1..10" step "put object parts 1..10"
inputFile <- mkRandFile mb5 inputFile <- mkRandFile mb5
h <- liftIO $ SIO.openBinaryFile inputFile SIO.ReadMode h <- liftIO $ SIO.openBinaryFile inputFile SIO.ReadMode
forM_ [1..10] $ \pnum -> forM_ [1 .. 10] $ \pnum ->
putObjectPart bucket object uid pnum [] $ PayloadH h 0 mb5 putObjectPart bucket object uid pnum [] $ PayloadH h 0 mb5
step "fetch list parts" step "fetch list parts"
@ -479,29 +555,46 @@ presignedUrlFunTest = funTestWithBucket "presigned Url tests" $
mgr <- liftIO $ NC.newManager NC.tlsManagerSettings mgr <- liftIO $ NC.newManager NC.tlsManagerSettings
step "PUT object presigned URL - makePresignedUrl" step "PUT object presigned URL - makePresignedUrl"
putUrl <- makePresignedUrl 3600 HT.methodPut (Just bucket) putUrl <-
(Just obj) (Just "us-east-1") [] [] makePresignedUrl
3600
HT.methodPut
(Just bucket)
(Just obj)
(Just "us-east-1")
[]
[]
let size1 = 1000 :: Int64 let size1 = 1000 :: Int64
inputFile <- mkRandFile size1 inputFile <- mkRandFile size1
-- attempt to upload using the presigned URL -- attempt to upload using the presigned URL
putResp <- putR size1 inputFile mgr putUrl putResp <- putR size1 inputFile mgr putUrl
liftIO $ (NC.responseStatus putResp == HT.status200) @? liftIO $
"presigned PUT failed" (NC.responseStatus putResp == HT.status200)
@? "presigned PUT failed"
step "GET object presigned URL - makePresignedUrl" step "GET object presigned URL - makePresignedUrl"
getUrl <- makePresignedUrl 3600 HT.methodGet (Just bucket) getUrl <-
(Just obj) (Just "us-east-1") [] [] makePresignedUrl
3600
HT.methodGet
(Just bucket)
(Just obj)
(Just "us-east-1")
[]
[]
getResp <- getR mgr getUrl getResp <- getR mgr getUrl
liftIO $ (NC.responseStatus getResp == HT.status200) @? liftIO $
"presigned GET failed" (NC.responseStatus getResp == HT.status200)
@? "presigned GET failed"
-- read content from file to compare with response above -- read content from file to compare with response above
bs <- C.runConduit $ CB.sourceFile inputFile C..| CB.sinkLbs bs <- C.runConduit $ CB.sourceFile inputFile C..| CB.sinkLbs
liftIO $ (bs == NC.responseBody getResp) @? liftIO $
"presigned put and get got mismatched data" (bs == NC.responseBody getResp)
@? "presigned put and get got mismatched data"
step "PUT object presigned - presignedPutObjectURL" step "PUT object presigned - presignedPutObjectURL"
putUrl2 <- presignedPutObjectUrl bucket obj2 604800 [] putUrl2 <- presignedPutObjectUrl bucket obj2 604800 []
@ -510,66 +603,71 @@ presignedUrlFunTest = funTestWithBucket "presigned Url tests" $
testFile <- mkRandFile size2 testFile <- mkRandFile size2
putResp2 <- putR size2 testFile mgr putUrl2 putResp2 <- putR size2 testFile mgr putUrl2
liftIO $ (NC.responseStatus putResp2 == HT.status200) @? liftIO $
"presigned PUT failed (presignedPutObjectUrl)" (NC.responseStatus putResp2 == HT.status200)
@? "presigned PUT failed (presignedPutObjectUrl)"
step "HEAD object presigned URL - presignedHeadObjectUrl" step "HEAD object presigned URL - presignedHeadObjectUrl"
headUrl <- presignedHeadObjectUrl bucket obj2 3600 [] headUrl <- presignedHeadObjectUrl bucket obj2 3600 []
headResp <- do let req = NC.parseRequest_ $ toS headUrl headResp <- do
let req = NC.parseRequest_ $ toS $ decodeUtf8 headUrl
NC.httpLbs (req {NC.method = HT.methodHead}) mgr NC.httpLbs (req {NC.method = HT.methodHead}) mgr
liftIO $ (NC.responseStatus headResp == HT.status200) @? liftIO $
"presigned HEAD failed (presignedHeadObjectUrl)" (NC.responseStatus headResp == HT.status200)
@? "presigned HEAD failed (presignedHeadObjectUrl)"
-- check that header info is accurate -- check that header info is accurate
let h = H.fromList $ NC.responseHeaders headResp let h = H.fromList $ NC.responseHeaders headResp
cLen = H.lookupDefault "0" HT.hContentLength h cLen = H.lookupDefault "0" HT.hContentLength h
liftIO $ (cLen == show size2) @? "Head req returned bad content length" liftIO $ (cLen == showBS size2) @? "Head req returned bad content length"
step "GET object presigned URL - presignedGetObjectUrl" step "GET object presigned URL - presignedGetObjectUrl"
getUrl2 <- presignedGetObjectUrl bucket obj2 3600 [] [] getUrl2 <- presignedGetObjectUrl bucket obj2 3600 [] []
getResp2 <- getR mgr getUrl2 getResp2 <- getR mgr getUrl2
liftIO $ (NC.responseStatus getResp2 == HT.status200) @? liftIO $
"presigned GET failed (presignedGetObjectUrl)" (NC.responseStatus getResp2 == HT.status200)
@? "presigned GET failed (presignedGetObjectUrl)"
-- read content from file to compare with response above -- read content from file to compare with response above
bs2 <- C.runConduit $ CB.sourceFile testFile C..| CB.sinkLbs bs2 <- C.runConduit $ CB.sourceFile testFile C..| CB.sinkLbs
liftIO $ (bs2 == NC.responseBody getResp2) @? liftIO $
"presigned put and get got mismatched data (presigned*Url)" (bs2 == NC.responseBody getResp2)
@? "presigned put and get got mismatched data (presigned*Url)"
mapM_ (removeObject bucket) [obj, obj2] mapM_ (removeObject bucket) [obj, obj2]
where where
putR size filePath mgr url = do putR size filePath mgr url = do
let req = NC.parseRequest_ $ toS url let req = NC.parseRequest_ $ toS $ decodeUtf8 url
let req' = req { NC.method = HT.methodPut let req' =
, NC.requestBody = NC.requestBodySource size $ req
CB.sourceFile filePath} { NC.method = HT.methodPut,
NC.requestBody =
NC.requestBodySource size $
CB.sourceFile filePath
}
NC.httpLbs req' mgr NC.httpLbs req' mgr
getR mgr url = do getR mgr url = do
let req = NC.parseRequest_ $ toS url let req = NC.parseRequest_ $ toS $ decodeUtf8 url
NC.httpLbs req mgr NC.httpLbs req mgr
presignedPostPolicyFunTest :: TestTree presignedPostPolicyFunTest :: TestTree
presignedPostPolicyFunTest = funTestWithBucket "Presigned Post Policy tests" $ presignedPostPolicyFunTest = funTestWithBucket "Presigned Post Policy tests" $
\step bucket -> do \step bucket -> do
step "presignedPostPolicy basic test" step "presignedPostPolicy basic test"
now <- liftIO $ Time.getCurrentTime now <- liftIO $ Time.getCurrentTime
let key = "presignedPostPolicyTest/myfile" let key = "presignedPostPolicyTest/myfile"
policyConds = [ ppCondBucket bucket policyConds =
, ppCondKey key [ ppCondBucket bucket,
, ppCondContentLengthRange 1 1000 ppCondKey key,
, ppCondContentType "application/octet-stream" ppCondContentLengthRange 1 1000,
, ppCondSuccessActionStatus 200 ppCondContentType "application/octet-stream",
ppCondSuccessActionStatus 200
] ]
expirationTime = Time.addUTCTime 3600 now expirationTime = Time.addUTCTime 3600 now
postPolicyE = newPostPolicy expirationTime policyConds postPolicyE = newPostPolicy expirationTime policyConds
size = 1000 :: Int64 size = 1000 :: Int64
inputFile <- mkRandFile size inputFile <- mkRandFile size
@ -580,15 +678,16 @@ presignedPostPolicyFunTest = funTestWithBucket "Presigned Post Policy tests" $
(url, formData) <- presignedPostPolicy postPolicy (url, formData) <- presignedPostPolicy postPolicy
-- liftIO (print url) >> liftIO (print formData) -- liftIO (print url) >> liftIO (print formData)
result <- liftIO $ postForm url formData inputFile result <- liftIO $ postForm url formData inputFile
liftIO $ (NC.responseStatus result == HT.status200) @? liftIO $
"presigned POST failed" (NC.responseStatus result == HT.status200)
@? "presigned POST failed"
mapM_ (removeObject bucket) [key] mapM_ (removeObject bucket) [key]
where where
postForm url formData inputFile = do postForm url formData inputFile = do
req <- NC.parseRequest $ toS url req <- NC.parseRequest $ toS $ decodeUtf8 url
let parts = map (\(x, y) -> Form.partBS x y) $ let parts =
map (\(x, y) -> Form.partBS x y) $
H.toList formData H.toList formData
parts' = parts ++ [Form.partFile "file" inputFile] parts' = parts ++ [Form.partFile "file" inputFile]
req' <- Form.formDataBody parts' req req' <- Form.formDataBody parts' req
@ -598,7 +697,6 @@ presignedPostPolicyFunTest = funTestWithBucket "Presigned Post Policy tests" $
bucketPolicyFunTest :: TestTree bucketPolicyFunTest :: TestTree
bucketPolicyFunTest = funTestWithBucket "Bucket Policy tests" $ bucketPolicyFunTest = funTestWithBucket "Bucket Policy tests" $
\step bucket -> do \step bucket -> do
step "bucketPolicy basic test - no policy exception" step "bucketPolicy basic test - no policy exception"
resE <- try $ getBucketPolicy bucket resE <- try $ getBucketPolicy bucket
case resE of case resE of
@ -631,10 +729,19 @@ bucketPolicyFunTest = funTestWithBucket "Bucket Policy tests" $
step "verify bucket policy: (2) get `myobject` anonymously" step "verify bucket policy: (2) get `myobject` anonymously"
connInfo <- asks mcConnInfo connInfo <- asks mcConnInfo
let proto = bool "http://" "https://" $ connectIsSecure connInfo let proto = bool "http://" "https://" $ connectIsSecure connInfo
url = BS.concat [proto, getHostAddr connInfo, "/", toS bucket, url =
"/", toS obj] BS.concat
respE <- liftIO $ (fmap (Right . toS) $ NC.simpleHttp $ toS url) `catch` [ proto,
(\(e :: NC.HttpException) -> return $ Left (show e :: Text)) getHostAddr connInfo,
"/",
toUtf8 bucket,
"/",
toUtf8 obj
]
respE <-
liftIO $
(fmap (Right . toStrictBS) $ NC.simpleHttp $ toS $ decodeUtf8 url)
`catch` (\(e :: NC.HttpException) -> return $ Left (show e :: Text))
case respE of case respE of
Left err -> liftIO $ assertFailure $ show err Left err -> liftIO $ assertFailure $ show err
Right s -> liftIO $ s @?= (BS.concat $ replicate 100 "c") Right s -> liftIO $ s @?= (BS.concat $ replicate 100 "c")
@ -658,8 +765,9 @@ multipartTest = funTestWithBucket "Multipart Tests" $
destFile <- mkRandFile 0 destFile <- mkRandFile 0
fGetObject bucket obj destFile defaultGetObjectOptions fGetObject bucket obj destFile defaultGetObjectOptions
gotSize <- withNewHandle destFile getFileSize gotSize <- withNewHandle destFile getFileSize
liftIO $ gotSize == Right (Just mb80) @? liftIO $
"Wrong file size of put file after getting" gotSize == Right (Just mb80)
@? "Wrong file size of put file after getting"
step "Cleanup actions" step "Cleanup actions"
removeObject bucket obj removeObject bucket obj
@ -679,13 +787,15 @@ multipartTest = funTestWithBucket "Multipart Tests" $
randFile <- mkRandFile kb5 randFile <- mkRandFile kb5
step "upload 2 parts" step "upload 2 parts"
forM_ [1,2] $ \partNum -> do forM_ [1, 2] $ \partNum -> do
h <- liftIO $ SIO.openBinaryFile randFile SIO.ReadMode h <- liftIO $ SIO.openBinaryFile randFile SIO.ReadMode
void $ putObjectPart bucket object uid partNum [] $ PayloadH h 0 kb5 void $ putObjectPart bucket object uid partNum [] $ PayloadH h 0 kb5
step "remove ongoing upload" step "remove ongoing upload"
removeIncompleteUpload bucket object removeIncompleteUpload bucket object
uploads <- C.runConduit $ listIncompleteUploads bucket (Just object) False uploads <-
C.runConduit $
listIncompleteUploads bucket (Just object) False
C..| sinkList C..| sinkList
liftIO $ (null uploads) @? "removeIncompleteUploads didn't complete successfully" liftIO $ (null uploads) @? "removeIncompleteUploads didn't complete successfully"
@ -698,8 +808,12 @@ putObjectContentTypeTest = funTestWithBucket "putObject contentType tests" $
step "create server object with content-type" step "create server object with content-type"
inputFile <- mkRandFile size1 inputFile <- mkRandFile size1
fPutObject bucket object inputFile defaultPutObjectOptions { fPutObject
pooContentType = Just "application/javascript" bucket
object
inputFile
defaultPutObjectOptions
{ pooContentType = Just "application/javascript"
} }
-- retrieve obj info to check -- retrieve obj info to check
@ -710,15 +824,22 @@ putObjectContentTypeTest = funTestWithBucket "putObject contentType tests" $
liftIO $ assertEqual "Content-Type did not match" (Just "application/javascript") (H.lookup "Content-Type" m) liftIO $ assertEqual "Content-Type did not match" (Just "application/javascript") (H.lookup "Content-Type" m)
step "upload object with content-encoding set to identity" step "upload object with content-encoding set to identity"
fPutObject bucket object inputFile defaultPutObjectOptions { fPutObject
pooContentEncoding = Just "identity" bucket
object
inputFile
defaultPutObjectOptions
{ pooContentEncoding = Just "identity"
} }
oiCE <- headObject bucket object [] oiCE <- headObject bucket object []
let m' = oiMetadata oiCE let m' = oiMetadata oiCE
step "Validate content-encoding" step "Validate content-encoding"
liftIO $ assertEqual "Content-Encoding did not match" (Just "identity") liftIO $
assertEqual
"Content-Encoding did not match"
(Just "identity")
(H.lookup "Content-Encoding" m') (H.lookup "Content-Encoding" m')
step "Cleanup actions" step "Cleanup actions"
@ -734,8 +855,12 @@ putObjectContentLanguageTest = funTestWithBucket "putObject contentLanguage test
step "create server object with content-language" step "create server object with content-language"
inputFile <- mkRandFile size1 inputFile <- mkRandFile size1
fPutObject bucket object inputFile defaultPutObjectOptions { fPutObject
pooContentLanguage = Just "en-US" bucket
object
inputFile
defaultPutObjectOptions
{ pooContentLanguage = Just "en-US"
} }
-- retrieve obj info to check -- retrieve obj info to check
@ -743,7 +868,10 @@ putObjectContentLanguageTest = funTestWithBucket "putObject contentLanguage test
let m = oiMetadata oi let m = oiMetadata oi
step "Validate content-language" step "Validate content-language"
liftIO $ assertEqual "content-language did not match" (Just "en-US") liftIO $
assertEqual
"content-language did not match"
(Just "en-US")
(H.lookup "Content-Language" m) (H.lookup "Content-Language" m)
step "Cleanup actions" step "Cleanup actions"
@ -759,9 +887,14 @@ putObjectUserMetadataTest = funTestWithBucket "putObject user-metadata test" $
step "create server object with usermetdata" step "create server object with usermetdata"
inputFile <- mkRandFile size1 inputFile <- mkRandFile size1
fPutObject bucket object inputFile defaultPutObjectOptions { fPutObject
pooUserMetadata = [ ("x-Amz-meta-mykey1", "myval1") bucket
, ("mykey2", "myval2") object
inputFile
defaultPutObjectOptions
{ pooUserMetadata =
[ ("x-Amz-meta-mykey1", "myval1"),
("mykey2", "myval2")
] ]
} }
@ -770,7 +903,8 @@ putObjectUserMetadataTest = funTestWithBucket "putObject user-metadata test" $
oi <- headObject bucket object [] oi <- headObject bucket object []
let m = oiUserMetadata oi let m = oiUserMetadata oi
-- need to do a case-insensitive comparison -- need to do a case-insensitive comparison
sortedMeta = sort $ map (\(k, v) -> (T.toLower k, T.toLower v)) $ sortedMeta =
sort $ map (\(k, v) -> (T.toLower k, T.toLower v)) $
H.toList m H.toList m
ref = sort [("mykey1", "myval1"), ("mykey2", "myval2")] ref = sort [("mykey1", "myval1"), ("mykey2", "myval2")]
@ -787,9 +921,14 @@ getObjectTest = funTestWithBucket "getObject test" $
size1 = 100 :: Int64 size1 = 100 :: Int64
inputFile <- mkRandFile size1 inputFile <- mkRandFile size1
fPutObject bucket object inputFile defaultPutObjectOptions { fPutObject
pooUserMetadata = [ ("x-Amz-meta-mykey1", "myval1") bucket
, ("mykey2", "myval2") object
inputFile
defaultPutObjectOptions
{ pooUserMetadata =
[ ("x-Amz-meta-mykey1", "myval1"),
("mykey2", "myval2")
] ]
} }
@ -798,7 +937,8 @@ getObjectTest = funTestWithBucket "getObject test" $
gor <- getObject bucket object defaultGetObjectOptions gor <- getObject bucket object defaultGetObjectOptions
let m = oiUserMetadata $ gorObjectInfo gor let m = oiUserMetadata $ gorObjectInfo gor
-- need to do a case-insensitive comparison -- need to do a case-insensitive comparison
sortedMeta = sort $ map (\(k, v) -> (T.toLower k, T.toLower v)) $ sortedMeta =
sort $ map (\(k, v) -> (T.toLower k, T.toLower v)) $
H.toList m H.toList m
ref = sort [("mykey1", "myval1"), ("mykey2", "myval2")] ref = sort [("mykey1", "myval1"), ("mykey2", "myval2")]
@ -827,12 +967,20 @@ putObjectStorageClassTest = funTestWithBucket "putObject storageClass tests" $
inputFile' <- mkRandFile size1 inputFile' <- mkRandFile size1
inputFile'' <- mkRandFile size0 inputFile'' <- mkRandFile size0
fPutObject bucket object inputFile defaultPutObjectOptions { fPutObject
pooStorageClass = Just "STANDARD" bucket
object
inputFile
defaultPutObjectOptions
{ pooStorageClass = Just "STANDARD"
} }
fPutObject bucket object' inputFile' defaultPutObjectOptions { fPutObject
pooStorageClass = Just "REDUCED_REDUNDANCY" bucket
object'
inputFile'
defaultPutObjectOptions
{ pooStorageClass = Just "REDUCED_REDUNDANCY"
} }
removeObject bucket object removeObject bucket object
@ -842,11 +990,20 @@ putObjectStorageClassTest = funTestWithBucket "putObject storageClass tests" $
let m' = oiMetadata oi' let m' = oiMetadata oi'
step "Validate x-amz-storage-class rrs" step "Validate x-amz-storage-class rrs"
liftIO $ assertEqual "storageClass did not match" (Just "REDUCED_REDUNDANCY") liftIO $
assertEqual
"storageClass did not match"
(Just "REDUCED_REDUNDANCY")
(H.lookup "X-Amz-Storage-Class" m') (H.lookup "X-Amz-Storage-Class" m')
fpE <- try $ fPutObject bucket object'' inputFile'' defaultPutObjectOptions { fpE <-
pooStorageClass = Just "INVALID_STORAGE_CLASS" try $
fPutObject
bucket
object''
inputFile''
defaultPutObjectOptions
{ pooStorageClass = Just "INVALID_STORAGE_CLASS"
} }
case fpE of case fpE of
Left exn -> liftIO $ exn @?= ServiceErr "InvalidStorageClass" "Invalid storage class." Left exn -> liftIO $ exn @?= ServiceErr "InvalidStorageClass" "Invalid storage class."
@ -869,7 +1026,7 @@ copyObjectTests = funTestWithBucket "copyObject related tests" $
fPutObject bucket object inputFile defaultPutObjectOptions fPutObject bucket object inputFile defaultPutObjectOptions
step "copy object" step "copy object"
let srcInfo = defaultSourceInfo { srcBucket = bucket, srcObject = object} let srcInfo = defaultSourceInfo {srcBucket = bucket, srcObject = object}
(etag, modTime) <- copyObjectSingle bucket objCopy srcInfo [] (etag, modTime) <- copyObjectSingle bucket objCopy srcInfo []
-- retrieve obj info to check -- retrieve obj info to check
@ -880,8 +1037,9 @@ copyObjectTests = funTestWithBucket "copyObject related tests" $
let isMTimeDiffOk = abs (diffUTCTime modTime t) < 1.0 let isMTimeDiffOk = abs (diffUTCTime modTime t) < 1.0
liftIO $ (s == size1 && e == etag && isMTimeDiffOk) @? liftIO $
"Copied object did not match expected." (s == size1 && e == etag && isMTimeDiffOk)
@? "Copied object did not match expected."
step "cleanup actions" step "cleanup actions"
removeObject bucket object removeObject bucket object
@ -902,12 +1060,18 @@ copyObjectTests = funTestWithBucket "copyObject related tests" $
liftIO $ (T.length uid > 0) @? "Got an empty multipartUpload Id." liftIO $ (T.length uid > 0) @? "Got an empty multipartUpload Id."
step "put object parts 1-3" step "put object parts 1-3"
let srcInfo' = defaultSourceInfo { srcBucket = bucket, srcObject = srcObj } let srcInfo' = defaultSourceInfo {srcBucket = bucket, srcObject = srcObj}
dstInfo' = defaultDestinationInfo { dstBucket = bucket, dstObject = copyObj } dstInfo' = defaultDestinationInfo {dstBucket = bucket, dstObject = copyObj}
parts <- forM [1..3] $ \p -> do parts <- forM [1 .. 3] $ \p -> do
(etag', _) <- copyObjectPart dstInfo' srcInfo'{ (etag', _) <-
srcRange = Just $ (,) ((p-1)*mb5) ((p-1)*mb5 + (mb5 - 1)) copyObjectPart
} uid (fromIntegral p) [] dstInfo'
srcInfo'
{ srcRange = Just $ (,) ((p -1) * mb5) ((p -1) * mb5 + (mb5 - 1))
}
uid
(fromIntegral p)
[]
return (fromIntegral p, etag') return (fromIntegral p, etag')
step "complete multipart" step "complete multipart"
@ -953,10 +1117,12 @@ copyObjectTests = funTestWithBucket "copyObject related tests" $
fPutObject bucket src inputFile' defaultPutObjectOptions fPutObject bucket src inputFile' defaultPutObjectOptions
step "copy last 10MiB of object" step "copy last 10MiB of object"
copyObject defaultDestinationInfo { dstBucket = bucket, dstObject = copyObj } defaultSourceInfo { copyObject
srcBucket = bucket defaultDestinationInfo {dstBucket = bucket, dstObject = copyObj}
, srcObject = src defaultSourceInfo
, srcRange = Just $ (,) (5 * 1024 * 1024) (size - 1) { srcBucket = bucket,
srcObject = src,
srcRange = Just $ (,) (5 * 1024 * 1024) (size - 1)
} }
step "verify uploaded object" step "verify uploaded object"
@ -973,16 +1139,17 @@ getNPutSSECTest =
-- disabled anyway. -- disabled anyway.
isTLSConn <- asks (connectIsSecure . mcConnInfo) isTLSConn <- asks (connectIsSecure . mcConnInfo)
if isTLSConn if isTLSConn
then do step "Make an encryption key" then do
key <- case mkSSECKey $ BS.pack [0..31] of step "Make an encryption key"
key <- case mkSSECKey $ BS.pack [0 .. 31] of
Nothing -> liftIO $ assertFailure "This should not happen" Nothing -> liftIO $ assertFailure "This should not happen"
Just k -> return k Just k -> return k
let mb1 = 1024*1024 let mb1 = 1024 * 1024
obj = "1" obj = "1"
step "Upload an object using the encryption key" step "Upload an object using the encryption key"
rFile <- mkRandFile mb1 rFile <- mkRandFile mb1
let putOpts = defaultPutObjectOptions { pooSSE = Just $ SSEC key } let putOpts = defaultPutObjectOptions {pooSSE = Just $ SSEC key}
fPutObject bucket obj rFile putOpts fPutObject bucket obj rFile putOpts
step "Stat object without key - should fail" step "Stat object without key - should fail"
@ -996,14 +1163,14 @@ getNPutSSECTest =
step "Get file and check length" step "Get file and check length"
dstFile <- mkRandFile 0 dstFile <- mkRandFile 0
let getOpts = defaultGetObjectOptions { gooSSECKey = Just key } let getOpts = defaultGetObjectOptions {gooSSECKey = Just key}
fGetObject bucket obj dstFile getOpts fGetObject bucket obj dstFile getOpts
gotSize <- withNewHandle dstFile getFileSize gotSize <- withNewHandle dstFile getFileSize
liftIO $ gotSize == Right (Just mb1) @? liftIO $
"Wrong file size of object when getting" gotSize == Right (Just mb1)
@? "Wrong file size of object when getting"
step "Cleanup" step "Cleanup"
deleteObject bucket obj deleteObject bucket obj
else step "Skipping encryption test as server is not using TLS" else step "Skipping encryption test as server is not using TLS"

View File

@ -15,88 +15,100 @@
-- --
module Network.Minio.API.Test module Network.Minio.API.Test
( bucketNameValidityTests ( bucketNameValidityTests,
, objectNameValidityTests objectNameValidityTests,
, parseServerInfoJSONTest parseServerInfoJSONTest,
, parseHealStatusTest parseHealStatusTest,
, parseHealStartRespTest parseHealStartRespTest,
) where )
where
import Data.Aeson (eitherDecode) import Data.Aeson (eitherDecode)
import Lib.Prelude
import Network.Minio.API
import Network.Minio.AdminAPI
import Test.Tasty import Test.Tasty
import Test.Tasty.HUnit import Test.Tasty.HUnit
import Lib.Prelude
import Network.Minio.AdminAPI
import Network.Minio.API
assertBool' :: Bool -> Assertion assertBool' :: Bool -> Assertion
assertBool' = assertBool "Test failed!" assertBool' = assertBool "Test failed!"
bucketNameValidityTests :: TestTree bucketNameValidityTests :: TestTree
bucketNameValidityTests = testGroup "Bucket Name Validity Tests" bucketNameValidityTests =
[ testCase "Too short 1" $ assertBool' $ not $ isValidBucketName "" testGroup
, testCase "Too short 2" $ assertBool' $ not $ isValidBucketName "ab" "Bucket Name Validity Tests"
, testCase "Too long 1" $ assertBool' $ not $ isValidBucketName "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" [ testCase "Too short 1" $ assertBool' $ not $ isValidBucketName "",
, testCase "Has upper case" $ assertBool' $ not $ isValidBucketName "ABCD" testCase "Too short 2" $ assertBool' $ not $ isValidBucketName "ab",
, testCase "Has punctuation" $ assertBool' $ not $ isValidBucketName "abc,2" testCase "Too long 1" $ assertBool' $ not $ isValidBucketName "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
, testCase "Has hyphen at end" $ assertBool' $ not $ isValidBucketName "abc-" testCase "Has upper case" $ assertBool' $ not $ isValidBucketName "ABCD",
, testCase "Has consecutive dot" $ assertBool' $ not $ isValidBucketName "abck..eedg" testCase "Has punctuation" $ assertBool' $ not $ isValidBucketName "abc,2",
, testCase "Looks like IP" $ assertBool' $ not $ isValidBucketName "10.0.0.1" testCase "Has hyphen at end" $ assertBool' $ not $ isValidBucketName "abc-",
, testCase "Valid bucket name 1" $ assertBool' $ isValidBucketName "abcd.pqeq.rea" testCase "Has consecutive dot" $ assertBool' $ not $ isValidBucketName "abck..eedg",
, testCase "Valid bucket name 2" $ assertBool' $ isValidBucketName "abcdedgh1d" testCase "Looks like IP" $ assertBool' $ not $ isValidBucketName "10.0.0.1",
, testCase "Valid bucket name 3" $ assertBool' $ isValidBucketName "abc-de-dg-h1d" testCase "Valid bucket name 1" $ assertBool' $ isValidBucketName "abcd.pqeq.rea",
testCase "Valid bucket name 2" $ assertBool' $ isValidBucketName "abcdedgh1d",
testCase "Valid bucket name 3" $ assertBool' $ isValidBucketName "abc-de-dg-h1d"
] ]
objectNameValidityTests :: TestTree objectNameValidityTests :: TestTree
objectNameValidityTests = testGroup "Object Name Validity Tests" objectNameValidityTests =
[ testCase "Empty name" $ assertBool' $ not $ isValidObjectName "" testGroup
, testCase "Has unicode characters" $ assertBool' $ isValidObjectName "日本国" "Object Name Validity Tests"
[ testCase "Empty name" $ assertBool' $ not $ isValidObjectName "",
testCase "Has unicode characters" $ assertBool' $ isValidObjectName "日本国"
] ]
parseServerInfoJSONTest :: TestTree parseServerInfoJSONTest :: TestTree
parseServerInfoJSONTest = testGroup "Parse MinIO Admin API ServerInfo JSON test" $ parseServerInfoJSONTest =
map (\(tName, tDesc, tfn, tVal) -> testCase tName $ assertBool tDesc $ testGroup "Parse MinIO Admin API ServerInfo JSON test" $
tfn (eitherDecode tVal :: Either [Char] [ServerInfo])) testCases map
( \(tName, tDesc, tfn, tVal) ->
testCase tName $ assertBool tDesc $
tfn (eitherDecode tVal :: Either [Char] [ServerInfo])
)
testCases
where where
testCases = [ ("FSBackend", "Verify server info json parsing for FS backend", isRight, fsJSON) testCases =
, ("Erasure Backend", "Verify server info json parsing for Erasure backend", isRight, erasureJSON) [ ("FSBackend", "Verify server info json parsing for FS backend", isRight, fsJSON),
, ("Unknown Backend", "Verify server info json parsing for invalid backend", isLeft, invalidJSON) ("Erasure Backend", "Verify server info json parsing for Erasure backend", isRight, erasureJSON),
("Unknown Backend", "Verify server info json parsing for invalid backend", isLeft, invalidJSON)
] ]
fsJSON = "[{\"error\":\"\",\"addr\":\"192.168.1.218:9000\",\"data\":{\"storage\":{\"Used\":20530,\"Backend\":{\"Type\":1,\"OnlineDisks\":0,\"OfflineDisks\":0,\"StandardSCData\":0,\"StandardSCParity\":0,\"RRSCData\":0,\"RRSCParity\":0,\"Sets\":null}},\"network\":{\"transferred\":808,\"received\":1160},\"http\":{\"totalHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalGETs\":{\"count\":1,\"avgDuration\":\"0s\"},\"successGETs\":{\"count\":1,\"avgDuration\":\"0s\"},\"totalPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"}},\"server\":{\"uptime\":5992503019270,\"version\":\"DEVELOPMENT.GOGET\",\"commitID\":\"DEVELOPMENT.GOGET\",\"region\":\"\",\"sqsARN\":[]}}}]" fsJSON = "[{\"error\":\"\",\"addr\":\"192.168.1.218:9000\",\"data\":{\"storage\":{\"Used\":20530,\"Backend\":{\"Type\":1,\"OnlineDisks\":0,\"OfflineDisks\":0,\"StandardSCData\":0,\"StandardSCParity\":0,\"RRSCData\":0,\"RRSCParity\":0,\"Sets\":null}},\"network\":{\"transferred\":808,\"received\":1160},\"http\":{\"totalHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalGETs\":{\"count\":1,\"avgDuration\":\"0s\"},\"successGETs\":{\"count\":1,\"avgDuration\":\"0s\"},\"totalPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"}},\"server\":{\"uptime\":5992503019270,\"version\":\"DEVELOPMENT.GOGET\",\"commitID\":\"DEVELOPMENT.GOGET\",\"region\":\"\",\"sqsARN\":[]}}}]"
erasureJSON = "[{\"error\":\"\",\"addr\":\"192.168.1.218:9000\",\"data\":{\"storage\":{\"Used\":83084,\"Backend\":{\"Type\":2,\"OnlineDisks\":4,\"OfflineDisks\":0,\"StandardSCData\":2,\"StandardSCParity\":2,\"RRSCData\":2,\"RRSCParity\":2,\"Sets\":[[{\"uuid\":\"16ec6f2c-9197-4787-904a-36bb2c2683f8\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"4052e086-ef99-4aa5-ae2b-8e27559432f6\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"d0639950-ddd3-45b0-93ca-fd86f5d79f72\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"30ec68c0-37e1-4592-82c1-26b143c0ac10\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]]}},\"network\":{\"transferred\":404,\"received\":0},\"http\":{\"totalHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalGETs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successGETs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"}},\"server\":{\"uptime\":2738903073,\"version\":\"DEVELOPMENT.GOGET\",\"commitID\":\"DEVELOPMENT.GOGET\",\"region\":\"\",\"sqsARN\":[]}}}]" erasureJSON = "[{\"error\":\"\",\"addr\":\"192.168.1.218:9000\",\"data\":{\"storage\":{\"Used\":83084,\"Backend\":{\"Type\":2,\"OnlineDisks\":4,\"OfflineDisks\":0,\"StandardSCData\":2,\"StandardSCParity\":2,\"RRSCData\":2,\"RRSCParity\":2,\"Sets\":[[{\"uuid\":\"16ec6f2c-9197-4787-904a-36bb2c2683f8\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"4052e086-ef99-4aa5-ae2b-8e27559432f6\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"d0639950-ddd3-45b0-93ca-fd86f5d79f72\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"30ec68c0-37e1-4592-82c1-26b143c0ac10\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]]}},\"network\":{\"transferred\":404,\"received\":0},\"http\":{\"totalHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalGETs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successGETs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"}},\"server\":{\"uptime\":2738903073,\"version\":\"DEVELOPMENT.GOGET\",\"commitID\":\"DEVELOPMENT.GOGET\",\"region\":\"\",\"sqsARN\":[]}}}]"
invalidJSON = "[{\"error\":\"\",\"addr\":\"192.168.1.218:9000\",\"data\":{\"storage\":{\"Used\":83084,\"Backend\":{\"Type\":42,\"OnlineDisks\":4,\"OfflineDisks\":0,\"StandardSCData\":2,\"StandardSCParity\":2,\"RRSCData\":2,\"RRSCParity\":2,\"Sets\":[[{\"uuid\":\"16ec6f2c-9197-4787-904a-36bb2c2683f8\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"4052e086-ef99-4aa5-ae2b-8e27559432f6\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"d0639950-ddd3-45b0-93ca-fd86f5d79f72\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"30ec68c0-37e1-4592-82c1-26b143c0ac10\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]]}},\"network\":{\"transferred\":404,\"received\":0},\"http\":{\"totalHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalGETs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successGETs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"}},\"server\":{\"uptime\":2738903073,\"version\":\"DEVELOPMENT.GOGET\",\"commitID\":\"DEVELOPMENT.GOGET\",\"region\":\"\",\"sqsARN\":[]}}}]" invalidJSON = "[{\"error\":\"\",\"addr\":\"192.168.1.218:9000\",\"data\":{\"storage\":{\"Used\":83084,\"Backend\":{\"Type\":42,\"OnlineDisks\":4,\"OfflineDisks\":0,\"StandardSCData\":2,\"StandardSCParity\":2,\"RRSCData\":2,\"RRSCParity\":2,\"Sets\":[[{\"uuid\":\"16ec6f2c-9197-4787-904a-36bb2c2683f8\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"4052e086-ef99-4aa5-ae2b-8e27559432f6\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"d0639950-ddd3-45b0-93ca-fd86f5d79f72\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"30ec68c0-37e1-4592-82c1-26b143c0ac10\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]]}},\"network\":{\"transferred\":404,\"received\":0},\"http\":{\"totalHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalGETs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successGETs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"}},\"server\":{\"uptime\":2738903073,\"version\":\"DEVELOPMENT.GOGET\",\"commitID\":\"DEVELOPMENT.GOGET\",\"region\":\"\",\"sqsARN\":[]}}}]"
parseHealStatusTest :: TestTree parseHealStatusTest :: TestTree
parseHealStatusTest = testGroup "Parse MinIO Admin API HealStatus JSON test" $ parseHealStatusTest =
map (\(tName, tDesc, tfn, tVal) -> testCase tName $ assertBool tDesc $ testGroup "Parse MinIO Admin API HealStatus JSON test" $
tfn (eitherDecode tVal :: Either [Char] HealStatus)) testCases map
( \(tName, tDesc, tfn, tVal) ->
testCase tName $ assertBool tDesc $
tfn (eitherDecode tVal :: Either [Char] HealStatus)
)
testCases
where where
testCases = [ ("Good", "Verify heal result item for erasure backend", isRight, erasureJSON') testCases =
, ("Corrupted", "Verify heal result item for erasure backend", isLeft, invalidJSON') [ ("Good", "Verify heal result item for erasure backend", isRight, erasureJSON'),
, ("Incorrect Value", "Verify heal result item for erasure backend", isLeft, invalidItemType) ("Corrupted", "Verify heal result item for erasure backend", isLeft, invalidJSON'),
("Incorrect Value", "Verify heal result item for erasure backend", isLeft, invalidItemType)
] ]
erasureJSON' = "{\"Summary\":\"finished\",\"StartTime\":\"2018-06-05T08:09:47.644465513Z\",\"NumDisks\":4,\"Settings\":{\"recursive\":false,\"dryRun\":false},\"Items\":[{\"resultId\":1,\"type\":\"metadata\",\"bucket\":\"\",\"object\":\"\",\"detail\":\"disk-format\",\"diskCount\":4,\"setCount\":1,\"before\":{\"drives\":[{\"uuid\":\"c3487166-b8a4-481a-b1e7-fb9b249e2500\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"55a6e787-184f-4e4c-bf09-03dcada658a9\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"f035d8c3-fca1-4407-b89c-38c2bcf4a641\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"4f8b79d3-db90-4c1d-87c2-35a28b0d9a13\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]},\"after\":{\"drives\":[{\"uuid\":\"c3487166-b8a4-481a-b1e7-fb9b249e2500\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"55a6e787-184f-4e4c-bf09-03dcada658a9\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"f035d8c3-fca1-4407-b89c-38c2bcf4a641\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"4f8b79d3-db90-4c1d-87c2-35a28b0d9a13\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]},\"objectSize\":0}]}" erasureJSON' = "{\"Summary\":\"finished\",\"StartTime\":\"2018-06-05T08:09:47.644465513Z\",\"NumDisks\":4,\"Settings\":{\"recursive\":false,\"dryRun\":false},\"Items\":[{\"resultId\":1,\"type\":\"metadata\",\"bucket\":\"\",\"object\":\"\",\"detail\":\"disk-format\",\"diskCount\":4,\"setCount\":1,\"before\":{\"drives\":[{\"uuid\":\"c3487166-b8a4-481a-b1e7-fb9b249e2500\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"55a6e787-184f-4e4c-bf09-03dcada658a9\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"f035d8c3-fca1-4407-b89c-38c2bcf4a641\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"4f8b79d3-db90-4c1d-87c2-35a28b0d9a13\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]},\"after\":{\"drives\":[{\"uuid\":\"c3487166-b8a4-481a-b1e7-fb9b249e2500\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"55a6e787-184f-4e4c-bf09-03dcada658a9\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"f035d8c3-fca1-4407-b89c-38c2bcf4a641\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"4f8b79d3-db90-4c1d-87c2-35a28b0d9a13\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]},\"objectSize\":0}]}"
invalidJSON' = "{\"Summary\":\"finished\",\"StartTime\":\"2018-06-05T08:09:47.644465513Z\",\"NumDisks\":4,\"Settings\":{\"recursive\":false,\"dryRun\":false},\"Items\":[{\"resultId\":1,\"type\":\"metadata\",\"bucket\":\"\",\"object\":\"\",\"detail\":\"disk-format\",\"diskCount\":4,\"setCount\":1,\"before\":{\"drives\":[{\"uuid\":\"c3487166-b8a4-481a-b1e7-fb9b249e2500\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"55a6e787-184f-4e4c-bf09-03dcada658a9\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"f035d8c3-fca1-4407-b89c-38c2bcf4a641\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"4f8b79d3-db90-4c1d-87c2-35a28b0d9a13\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]},\"after\":{\"drives\":[{\"uuid\":\"c3487166-b8a4-481a-b1e7-fb9b249e2500\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"55a6e787-184f-4e4c-bf09-03dcada658a9\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"f035d8c3-fca1-4407-b89c-38c2bcf4a641\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"4f8b79d3-db90-4c1d-87c2-35a28b0d9a13\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]},\"objectSize\":0}]" invalidJSON' = "{\"Summary\":\"finished\",\"StartTime\":\"2018-06-05T08:09:47.644465513Z\",\"NumDisks\":4,\"Settings\":{\"recursive\":false,\"dryRun\":false},\"Items\":[{\"resultId\":1,\"type\":\"metadata\",\"bucket\":\"\",\"object\":\"\",\"detail\":\"disk-format\",\"diskCount\":4,\"setCount\":1,\"before\":{\"drives\":[{\"uuid\":\"c3487166-b8a4-481a-b1e7-fb9b249e2500\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"55a6e787-184f-4e4c-bf09-03dcada658a9\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"f035d8c3-fca1-4407-b89c-38c2bcf4a641\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"4f8b79d3-db90-4c1d-87c2-35a28b0d9a13\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]},\"after\":{\"drives\":[{\"uuid\":\"c3487166-b8a4-481a-b1e7-fb9b249e2500\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"55a6e787-184f-4e4c-bf09-03dcada658a9\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"f035d8c3-fca1-4407-b89c-38c2bcf4a641\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"4f8b79d3-db90-4c1d-87c2-35a28b0d9a13\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]},\"objectSize\":0}]"
invalidItemType = "{\"Summary\":\"finished\",\"StartTime\":\"2018-06-05T08:09:47.644465513Z\",\"NumDisks\":4,\"Settings\":{\"recursive\":false,\"dryRun\":false},\"Items\":[{\"resultId\":1,\"type\":\"hello\",\"bucket\":\"\",\"object\":\"\",\"detail\":\"disk-format\",\"diskCount\":4,\"setCount\":1,\"before\":{\"drives\":[{\"uuid\":\"c3487166-b8a4-481a-b1e7-fb9b249e2500\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"55a6e787-184f-4e4c-bf09-03dcada658a9\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"f035d8c3-fca1-4407-b89c-38c2bcf4a641\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"4f8b79d3-db90-4c1d-87c2-35a28b0d9a13\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]},\"after\":{\"drives\":[{\"uuid\":\"c3487166-b8a4-481a-b1e7-fb9b249e2500\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"55a6e787-184f-4e4c-bf09-03dcada658a9\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"f035d8c3-fca1-4407-b89c-38c2bcf4a641\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"4f8b79d3-db90-4c1d-87c2-35a28b0d9a13\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]},\"objectSize\":0}]}" invalidItemType = "{\"Summary\":\"finished\",\"StartTime\":\"2018-06-05T08:09:47.644465513Z\",\"NumDisks\":4,\"Settings\":{\"recursive\":false,\"dryRun\":false},\"Items\":[{\"resultId\":1,\"type\":\"hello\",\"bucket\":\"\",\"object\":\"\",\"detail\":\"disk-format\",\"diskCount\":4,\"setCount\":1,\"before\":{\"drives\":[{\"uuid\":\"c3487166-b8a4-481a-b1e7-fb9b249e2500\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"55a6e787-184f-4e4c-bf09-03dcada658a9\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"f035d8c3-fca1-4407-b89c-38c2bcf4a641\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"4f8b79d3-db90-4c1d-87c2-35a28b0d9a13\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]},\"after\":{\"drives\":[{\"uuid\":\"c3487166-b8a4-481a-b1e7-fb9b249e2500\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"55a6e787-184f-4e4c-bf09-03dcada658a9\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"f035d8c3-fca1-4407-b89c-38c2bcf4a641\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"4f8b79d3-db90-4c1d-87c2-35a28b0d9a13\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]},\"objectSize\":0}]}"
parseHealStartRespTest :: TestTree parseHealStartRespTest :: TestTree
parseHealStartRespTest = testGroup "Parse MinIO Admin API HealStartResp JSON test" $ parseHealStartRespTest =
map (\(tName, tDesc, tfn, tVal) -> testCase tName $ assertBool tDesc $ testGroup "Parse MinIO Admin API HealStartResp JSON test" $
tfn (eitherDecode tVal :: Either [Char] HealStartResp)) testCases map
( \(tName, tDesc, tfn, tVal) ->
testCase tName $ assertBool tDesc $
tfn (eitherDecode tVal :: Either [Char] HealStartResp)
)
testCases
where where
testCases = [ ("Good", "Verify heal start response for erasure backend", isRight, hsrJSON) testCases =
, ("Missing Token", "Verify heal start response for erasure backend", isLeft, missingTokenJSON) [ ("Good", "Verify heal start response for erasure backend", isRight, hsrJSON),
("Missing Token", "Verify heal start response for erasure backend", isLeft, missingTokenJSON)
] ]
hsrJSON = "{\"clientToken\":\"3a3aca49-77dd-4b78-bba7-0978f119b23e\",\"clientAddress\":\"127.0.0.1\",\"startTime\":\"2018-06-05T08:09:47.644394493Z\"}" hsrJSON = "{\"clientToken\":\"3a3aca49-77dd-4b78-bba7-0978f119b23e\",\"clientAddress\":\"127.0.0.1\",\"startTime\":\"2018-06-05T08:09:47.644394493Z\"}"
missingTokenJSON = "{\"clientAddress\":\"127.0.0.1\",\"startTime\":\"2018-06-05T08:09:47.644394493Z\"}" missingTokenJSON = "{\"clientAddress\":\"127.0.0.1\",\"startTime\":\"2018-06-05T08:09:47.644394493Z\"}"

View File

@ -15,21 +15,21 @@
-- --
module Network.Minio.JsonParser.Test module Network.Minio.JsonParser.Test
( ( jsonParserTests,
jsonParserTests )
) where where
import Lib.Prelude
import Network.Minio.Errors
import Network.Minio.JsonParser
import Test.Tasty import Test.Tasty
import Test.Tasty.HUnit import Test.Tasty.HUnit
import UnliftIO (MonadUnliftIO) import UnliftIO (MonadUnliftIO)
import Lib.Prelude
import Network.Minio.Errors
import Network.Minio.JsonParser
jsonParserTests :: TestTree jsonParserTests :: TestTree
jsonParserTests = testGroup "JSON Parser Tests" jsonParserTests =
testGroup
"JSON Parser Tests"
[ testCase "Test parseErrResponseJSON" testParseErrResponseJSON [ testCase "Test parseErrResponseJSON" testParseErrResponseJSON
] ]
@ -43,22 +43,21 @@ testParseErrResponseJSON :: Assertion
testParseErrResponseJSON = do testParseErrResponseJSON = do
-- 1. Test parsing of an invalid error json. -- 1. Test parsing of an invalid error json.
parseResE <- tryValidationErr $ parseErrResponseJSON "ClearlyInvalidJSON" parseResE <- tryValidationErr $ parseErrResponseJSON "ClearlyInvalidJSON"
when (isRight parseResE) $ when (isRight parseResE)
assertFailure $ "Parsing should have failed => " ++ show parseResE $ assertFailure
$ "Parsing should have failed => " ++ show parseResE
forM_ cases $ \(jsondata, sErr) -> do forM_ cases $ \(jsondata, sErr) -> do
parseErr <- tryValidationErr $ parseErrResponseJSON jsondata parseErr <- tryValidationErr $ parseErrResponseJSON jsondata
either assertValidationErr (@?= sErr) parseErr either assertValidationErr (@?= sErr) parseErr
where where
cases = [ cases =
-- 2. Test parsing of a valid error json. [ -- 2. Test parsing of a valid error json.
("{\"Code\":\"InvalidAccessKeyId\",\"Message\":\"The access key ID you provided does not exist in our records.\",\"Key\":\"\",\"BucketName\":\"\",\"Resource\":\"/minio/admin/v1/info\",\"RequestId\":\"3L137\",\"HostId\":\"3L137\"}", ( "{\"Code\":\"InvalidAccessKeyId\",\"Message\":\"The access key ID you provided does not exist in our records.\",\"Key\":\"\",\"BucketName\":\"\",\"Resource\":\"/minio/admin/v1/info\",\"RequestId\":\"3L137\",\"HostId\":\"3L137\"}",
ServiceErr "InvalidAccessKeyId" "The access key ID you provided does not exist in our records." ServiceErr "InvalidAccessKeyId" "The access key ID you provided does not exist in our records."
) ),
,
-- 3. Test parsing of a valid, empty Resource. -- 3. Test parsing of a valid, empty Resource.
("{\"Code\":\"SignatureDoesNotMatch\",\"Message\":\"The request signature we calculated does not match the signature you provided. Check your key and signing method.\",\"Key\":\"\",\"BucketName\":\"\",\"Resource\":\"/minio/admin/v1/info\",\"RequestId\":\"3L137\",\"HostId\":\"3L137\"}", ( "{\"Code\":\"SignatureDoesNotMatch\",\"Message\":\"The request signature we calculated does not match the signature you provided. Check your key and signing method.\",\"Key\":\"\",\"BucketName\":\"\",\"Resource\":\"/minio/admin/v1/info\",\"RequestId\":\"3L137\",\"HostId\":\"3L137\"}",
ServiceErr "SignatureDoesNotMatch" "The request signature we calculated does not match the signature you provided. Check your key and signing method." ServiceErr "SignatureDoesNotMatch" "The request signature we calculated does not match the signature you provided. Check your key and signing method."
) )
] ]

View File

@ -15,18 +15,19 @@
-- --
module Network.Minio.TestHelpers module Network.Minio.TestHelpers
( runTestNS ( runTestNS,
) where )
where
import Network.Minio.Data
import Lib.Prelude import Lib.Prelude
import Network.Minio.Data
newtype TestNS = TestNS { testNamespace :: Text } newtype TestNS = TestNS {testNamespace :: Text}
instance HasSvcNamespace TestNS where instance HasSvcNamespace TestNS where
getSvcNamespace = testNamespace getSvcNamespace = testNamespace
runTestNS :: ReaderT TestNS m a -> m a runTestNS :: ReaderT TestNS m a -> m a
runTestNS = flip runReaderT $ runTestNS =
flip runReaderT $
TestNS "http://s3.amazonaws.com/doc/2006-03-01/" TestNS "http://s3.amazonaws.com/doc/2006-03-01/"

View File

@ -15,19 +15,19 @@
-- --
module Network.Minio.Utils.Test module Network.Minio.Utils.Test
( ( limitedMapConcurrentlyTests,
limitedMapConcurrentlyTests )
) where where
import Lib.Prelude
import Network.Minio.Utils
import Test.Tasty import Test.Tasty
import Test.Tasty.HUnit import Test.Tasty.HUnit
import Lib.Prelude
import Network.Minio.Utils
limitedMapConcurrentlyTests :: TestTree limitedMapConcurrentlyTests :: TestTree
limitedMapConcurrentlyTests = testGroup "limitedMapConcurrently Tests" limitedMapConcurrentlyTests =
testGroup
"limitedMapConcurrently Tests"
[ testCase "Test with various thread counts" testLMC [ testCase "Test with various thread counts" testLMC
] ]
@ -35,13 +35,12 @@ testLMC :: Assertion
testLMC = do testLMC = do
let maxNum = 50 let maxNum = 50
-- test with thread count of 1 to 2*maxNum -- test with thread count of 1 to 2*maxNum
forM_ [1..(2*maxNum)] $ \threads -> do forM_ [1 .. (2 * maxNum)] $ \threads -> do
res <- limitedMapConcurrently threads compute [1..maxNum] res <- limitedMapConcurrently threads compute [1 .. maxNum]
sum res @?= overallResultCheck maxNum sum res @?= overallResultCheck maxNum
where where
-- simple function to run in each thread -- simple function to run in each thread
compute :: Int -> IO Int compute :: Int -> IO Int
compute n = return $ sum [1..n] compute n = return $ sum [1 .. n]
-- function to check overall result -- function to check overall result
overallResultCheck n = sum $ map (\t -> (t * (t+1)) `div` 2) [1..n] overallResultCheck n = sum $ map (\t -> (t * (t + 1)) `div` 2) [1 .. n]

View File

@ -13,29 +13,30 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE QuasiQuotes #-}
module Network.Minio.XmlGenerator.Test
( xmlGeneratorTests
) where
import Test.Tasty module Network.Minio.XmlGenerator.Test
import Test.Tasty.HUnit ( xmlGeneratorTests,
import Text.RawString.QQ (r) )
where
import Lib.Prelude import Lib.Prelude
import Network.Minio.Data import Network.Minio.Data
import Network.Minio.TestHelpers import Network.Minio.TestHelpers
import Network.Minio.XmlGenerator import Network.Minio.XmlGenerator
import Network.Minio.XmlParser (parseNotification) import Network.Minio.XmlParser (parseNotification)
import Test.Tasty
import Test.Tasty.HUnit
import Text.RawString.QQ (r)
xmlGeneratorTests :: TestTree xmlGeneratorTests :: TestTree
xmlGeneratorTests = testGroup "XML Generator Tests" xmlGeneratorTests =
[ testCase "Test mkCreateBucketConfig" testMkCreateBucketConfig testGroup
, testCase "Test mkCompleteMultipartUploadRequest" testMkCompleteMultipartUploadRequest "XML Generator Tests"
, testCase "Test mkPutNotificationRequest" testMkPutNotificationRequest [ testCase "Test mkCreateBucketConfig" testMkCreateBucketConfig,
, testCase "Test mkSelectRequest" testMkSelectRequest testCase "Test mkCompleteMultipartUploadRequest" testMkCompleteMultipartUploadRequest,
testCase "Test mkPutNotificationRequest" testMkPutNotificationRequest,
testCase "Test mkSelectRequest" testMkSelectRequest
] ]
testMkCreateBucketConfig :: Assertion testMkCreateBucketConfig :: Assertion
@ -44,7 +45,8 @@ testMkCreateBucketConfig = do
assertEqual "CreateBucketConfiguration xml should match: " expected $ assertEqual "CreateBucketConfiguration xml should match: " expected $
mkCreateBucketConfig ns "EU" mkCreateBucketConfig ns "EU"
where where
expected = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\ expected =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
\<CreateBucketConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\ \<CreateBucketConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
\<LocationConstraint>EU</LocationConstraint>\ \<LocationConstraint>EU</LocationConstraint>\
\</CreateBucketConfiguration>" \</CreateBucketConfiguration>"
@ -54,48 +56,63 @@ testMkCompleteMultipartUploadRequest =
assertEqual "completeMultipartUpload xml should match: " expected $ assertEqual "completeMultipartUpload xml should match: " expected $
mkCompleteMultipartUploadRequest [(1, "abc")] mkCompleteMultipartUploadRequest [(1, "abc")]
where where
expected = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\ expected =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
\<CompleteMultipartUpload>\ \<CompleteMultipartUpload>\
\<Part>\ \<Part>\
\<PartNumber>1</PartNumber><ETag>abc</ETag>\ \<PartNumber>1</PartNumber><ETag>abc</ETag>\
\</Part>\ \</Part>\
\</CompleteMultipartUpload>" \</CompleteMultipartUpload>"
testMkPutNotificationRequest :: Assertion testMkPutNotificationRequest :: Assertion
testMkPutNotificationRequest = testMkPutNotificationRequest =
forM_ cases $ \val -> do forM_ cases $ \val -> do
let ns = "http://s3.amazonaws.com/doc/2006-03-01/" let ns = "http://s3.amazonaws.com/doc/2006-03-01/"
result = toS $ mkPutNotificationRequest ns val result = fromStrictBS $ mkPutNotificationRequest ns val
ntf <- runExceptT $ runTestNS $ parseNotification result ntf <- runExceptT $ runTestNS $ parseNotification result
either (\_ -> assertFailure "XML Parse Error!") either
(@?= val) ntf (\_ -> assertFailure "XML Parse Error!")
(@?= val)
ntf
where where
cases = [ Notification [] cases =
[ Notification
[]
[ NotificationConfig [ NotificationConfig
"YjVkM2Y0YmUtNGI3NC00ZjQyLWEwNGItNDIyYWUxY2I0N2M4" "YjVkM2Y0YmUtNGI3NC00ZjQyLWEwNGItNDIyYWUxY2I0N2M4"
"arn:aws:sns:us-east-1:account-id:s3notificationtopic2" "arn:aws:sns:us-east-1:account-id:s3notificationtopic2"
[ReducedRedundancyLostObject, ObjectCreated] defaultFilter [ReducedRedundancyLostObject, ObjectCreated]
defaultFilter
] ]
[] [],
, Notification Notification
[ NotificationConfig [ NotificationConfig
"1" "arn:aws:sqs:us-west-2:444455556666:s3notificationqueue" "1"
"arn:aws:sqs:us-west-2:444455556666:s3notificationqueue"
[ObjectCreatedPut] [ObjectCreatedPut]
(Filter $ FilterKey $ FilterRules ( Filter $ FilterKey $
[ FilterRule "prefix" "images/" FilterRules
, FilterRule "suffix" ".jpg"]) [ FilterRule "prefix" "images/",
, NotificationConfig FilterRule "suffix" ".jpg"
"" "arn:aws:sqs:us-east-1:356671443308:s3notificationqueue" ]
[ObjectCreated] defaultFilter ),
NotificationConfig
""
"arn:aws:sqs:us-east-1:356671443308:s3notificationqueue"
[ObjectCreated]
defaultFilter
] ]
[ NotificationConfig [ NotificationConfig
"" "arn:aws:sns:us-east-1:356671443308:s3notificationtopic2" ""
[ReducedRedundancyLostObject] defaultFilter "arn:aws:sns:us-east-1:356671443308:s3notificationtopic2"
[ReducedRedundancyLostObject]
defaultFilter
] ]
[ NotificationConfig [ NotificationConfig
"ObjectCreatedEvents" "arn:aws:lambda:us-west-2:35667example:function:CreateThumbnail" "ObjectCreatedEvents"
[ObjectCreated] defaultFilter "arn:aws:lambda:us-west-2:35667example:function:CreateThumbnail"
[ObjectCreated]
defaultFilter
] ]
] ]
@ -103,41 +120,54 @@ testMkSelectRequest :: Assertion
testMkSelectRequest = mapM_ assertFn cases testMkSelectRequest = mapM_ assertFn cases
where where
assertFn (a, b) = assertEqual "selectRequest XML should match: " b $ mkSelectRequest a assertFn (a, b) = assertEqual "selectRequest XML should match: " b $ mkSelectRequest a
cases = [ ( SelectRequest "Select * from S3Object" SQL cases =
(InputSerialization (Just CompressionTypeGzip) [ ( SelectRequest
(InputFormatCSV $ fileHeaderInfo FileHeaderIgnore "Select * from S3Object"
<> recordDelimiter "\n" SQL
<> fieldDelimiter "," ( InputSerialization
<> quoteCharacter "\"" (Just CompressionTypeGzip)
<> quoteEscapeCharacter "\"" ( InputFormatCSV $
)) fileHeaderInfo FileHeaderIgnore
(OutputSerializationCSV $ quoteFields QuoteFieldsAsNeeded
<> recordDelimiter "\n" <> recordDelimiter "\n"
<> fieldDelimiter "," <> fieldDelimiter ","
<> quoteCharacter "\"" <> quoteCharacter "\""
<> quoteEscapeCharacter "\"" <> quoteEscapeCharacter "\""
) )
(Just False) )
, [r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>GZIP</CompressionType><CSV><QuoteCharacter>&#34;</QuoteCharacter><RecordDelimiter> ( OutputSerializationCSV $
quoteFields QuoteFieldsAsNeeded
<> recordDelimiter "\n"
<> fieldDelimiter ","
<> quoteCharacter "\""
<> quoteEscapeCharacter "\""
)
(Just False),
[r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>GZIP</CompressionType><CSV><QuoteCharacter>&#34;</QuoteCharacter><RecordDelimiter>
</RecordDelimiter><FileHeaderInfo>IGNORE</FileHeaderInfo><QuoteEscapeCharacter>&#34;</QuoteEscapeCharacter><FieldDelimiter>,</FieldDelimiter></CSV></InputSerialization><OutputSerialization><CSV><QuoteCharacter>&#34;</QuoteCharacter><QuoteFields>ASNEEDED</QuoteFields><RecordDelimiter> </RecordDelimiter><FileHeaderInfo>IGNORE</FileHeaderInfo><QuoteEscapeCharacter>&#34;</QuoteEscapeCharacter><FieldDelimiter>,</FieldDelimiter></CSV></InputSerialization><OutputSerialization><CSV><QuoteCharacter>&#34;</QuoteCharacter><QuoteFields>ASNEEDED</QuoteFields><RecordDelimiter>
</RecordDelimiter><QuoteEscapeCharacter>&#34;</QuoteEscapeCharacter><FieldDelimiter>,</FieldDelimiter></CSV></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|] </RecordDelimiter><QuoteEscapeCharacter>&#34;</QuoteEscapeCharacter><FieldDelimiter>,</FieldDelimiter></CSV></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|]
) ),
, ( setRequestProgressEnabled False $ ( setRequestProgressEnabled False
setInputCompressionType CompressionTypeGzip $ $ setInputCompressionType CompressionTypeGzip
selectRequest "Select * from S3Object" documentJsonInput $ selectRequest
(outputJSONFromRecordDelimiter "\n") "Select * from S3Object"
, [r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>GZIP</CompressionType><JSON><Type>DOCUMENT</Type></JSON></InputSerialization><OutputSerialization><JSON><RecordDelimiter> documentJsonInput
(outputJSONFromRecordDelimiter "\n"),
[r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>GZIP</CompressionType><JSON><Type>DOCUMENT</Type></JSON></InputSerialization><OutputSerialization><JSON><RecordDelimiter>
</RecordDelimiter></JSON></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|] </RecordDelimiter></JSON></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|]
) ),
, ( setRequestProgressEnabled False $ ( setRequestProgressEnabled False
setInputCompressionType CompressionTypeNone $ $ setInputCompressionType CompressionTypeNone
selectRequest "Select * from S3Object" defaultParquetInput $ selectRequest
(outputCSVFromProps $ quoteFields QuoteFieldsAsNeeded "Select * from S3Object"
defaultParquetInput
( outputCSVFromProps $
quoteFields QuoteFieldsAsNeeded
<> recordDelimiter "\n" <> recordDelimiter "\n"
<> fieldDelimiter "," <> fieldDelimiter ","
<> quoteCharacter "\"" <> quoteCharacter "\""
<> quoteEscapeCharacter "\"") <> quoteEscapeCharacter "\""
, [r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>NONE</CompressionType><Parquet/></InputSerialization><OutputSerialization><CSV><QuoteCharacter>&#34;</QuoteCharacter><QuoteFields>ASNEEDED</QuoteFields><RecordDelimiter> ),
[r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>NONE</CompressionType><Parquet/></InputSerialization><OutputSerialization><CSV><QuoteCharacter>&#34;</QuoteCharacter><QuoteFields>ASNEEDED</QuoteFields><RecordDelimiter>
</RecordDelimiter><QuoteEscapeCharacter>&#34;</QuoteEscapeCharacter><FieldDelimiter>,</FieldDelimiter></CSV></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|] </RecordDelimiter><QuoteEscapeCharacter>&#34;</QuoteEscapeCharacter><FieldDelimiter>,</FieldDelimiter></CSV></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|]
) )
] ]

View File

@ -13,38 +13,39 @@
-- See the License for the specific language governing permissions and -- See the License for the specific language governing permissions and
-- limitations under the License. -- limitations under the License.
-- --
{-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE QuasiQuotes #-}
module Network.Minio.XmlParser.Test module Network.Minio.XmlParser.Test
( xmlParserTests ( xmlParserTests,
) where )
where
import qualified Data.HashMap.Strict as H import qualified Data.HashMap.Strict as H
import Data.Time (fromGregorian) import Data.Time (fromGregorian)
import Lib.Prelude
import Network.Minio.Data
import Network.Minio.Errors
import Network.Minio.TestHelpers
import Network.Minio.XmlParser
import Test.Tasty import Test.Tasty
import Test.Tasty.HUnit import Test.Tasty.HUnit
import Text.RawString.QQ (r) import Text.RawString.QQ (r)
import UnliftIO (MonadUnliftIO) import UnliftIO (MonadUnliftIO)
import Lib.Prelude
import Network.Minio.Data
import Network.Minio.Errors
import Network.Minio.TestHelpers
import Network.Minio.XmlParser
xmlParserTests :: TestTree xmlParserTests :: TestTree
xmlParserTests = testGroup "XML Parser Tests" xmlParserTests =
[ testCase "Test parseLocation" testParseLocation testGroup
, testCase "Test parseNewMultipartUpload" testParseNewMultipartUpload "XML Parser Tests"
, testCase "Test parseListObjectsResponse" testParseListObjectsResult [ testCase "Test parseLocation" testParseLocation,
, testCase "Test parseListObjectsV1Response" testParseListObjectsV1Result testCase "Test parseNewMultipartUpload" testParseNewMultipartUpload,
, testCase "Test parseListUploadsresponse" testParseListIncompleteUploads testCase "Test parseListObjectsResponse" testParseListObjectsResult,
, testCase "Test parseCompleteMultipartUploadResponse" testParseCompleteMultipartUploadResponse testCase "Test parseListObjectsV1Response" testParseListObjectsV1Result,
, testCase "Test parseListPartsResponse" testParseListPartsResponse testCase "Test parseListUploadsresponse" testParseListIncompleteUploads,
, testCase "Test parseCopyObjectResponse" testParseCopyObjectResponse testCase "Test parseCompleteMultipartUploadResponse" testParseCompleteMultipartUploadResponse,
, testCase "Test parseNotification" testParseNotification testCase "Test parseListPartsResponse" testParseListPartsResponse,
, testCase "Test parseSelectProgress" testParseSelectProgress testCase "Test parseCopyObjectResponse" testParseCopyObjectResponse,
testCase "Test parseNotification" testParseNotification,
testCase "Test parseSelectProgress" testParseSelectProgress
] ]
tryValidationErr :: (MonadUnliftIO m) => m a -> m (Either MErrV a) tryValidationErr :: (MonadUnliftIO m) => m a -> m (Either MErrV a)
@ -61,35 +62,34 @@ testParseLocation :: Assertion
testParseLocation = do testParseLocation = do
-- 1. Test parsing of an invalid location constraint xml. -- 1. Test parsing of an invalid location constraint xml.
parseResE <- tryValidationErr $ parseLocation "ClearlyInvalidXml" parseResE <- tryValidationErr $ parseLocation "ClearlyInvalidXml"
when (isRight parseResE) $ when (isRight parseResE)
assertFailure $ "Parsing should have failed => " ++ show parseResE $ assertFailure
$ "Parsing should have failed => " ++ show parseResE
forM_ cases $ \(xmldata, expectedLocation) -> do forM_ cases $ \(xmldata, expectedLocation) -> do
parseLocE <- tryValidationErr $ parseLocation xmldata parseLocE <- tryValidationErr $ parseLocation xmldata
either assertValidtionErr (@?= expectedLocation) parseLocE either assertValidtionErr (@?= expectedLocation) parseLocE
where where
cases = [ cases =
-- 2. Test parsing of a valid location xml. [ -- 2. Test parsing of a valid location xml.
("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\ ( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
\<LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">EU</LocationConstraint>", \<LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">EU</LocationConstraint>",
"EU" "EU"
) ),
,
-- 3. Test parsing of a valid, empty location xml. -- 3. Test parsing of a valid, empty location xml.
("<LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>", ( "<LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"/>",
"us-east-1" "us-east-1"
) )
] ]
testParseNewMultipartUpload :: Assertion testParseNewMultipartUpload :: Assertion
testParseNewMultipartUpload = do testParseNewMultipartUpload = do
forM_ cases $ \(xmldata, expectedUploadId) -> do forM_ cases $ \(xmldata, expectedUploadId) -> do
parsedUploadIdE <- tryValidationErr $ runTestNS $ parseNewMultipartUpload xmldata parsedUploadIdE <- tryValidationErr $ runTestNS $ parseNewMultipartUpload xmldata
eitherValidationErr parsedUploadIdE (@?= expectedUploadId) eitherValidationErr parsedUploadIdE (@?= expectedUploadId)
where where
cases = [ cases =
("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\ [ ( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
\<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\ \<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
\ <Bucket>example-bucket</Bucket>\ \ <Bucket>example-bucket</Bucket>\
\ <Key>example-object</Key>\ \ <Key>example-object</Key>\
@ -97,7 +97,7 @@ testParseNewMultipartUpload = do
\</InitiateMultipartUploadResult>", \</InitiateMultipartUploadResult>",
"VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA" "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA"
), ),
("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\ ( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
\<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\ \<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
\ <Bucket>example-bucket</Bucket>\ \ <Bucket>example-bucket</Bucket>\
\ <Key>example-object</Key>\ \ <Key>example-object</Key>\
@ -109,8 +109,8 @@ testParseNewMultipartUpload = do
testParseListObjectsResult :: Assertion testParseListObjectsResult :: Assertion
testParseListObjectsResult = do testParseListObjectsResult = do
let let xmldata =
xmldata = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
\<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\ \<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
\<Name>bucket</Name>\ \<Name>bucket</Name>\
\<Prefix/>\ \<Prefix/>\
@ -126,7 +126,6 @@ testParseListObjectsResult = do
\<StorageClass>STANDARD</StorageClass>\ \<StorageClass>STANDARD</StorageClass>\
\</Contents>\ \</Contents>\
\</ListBucketResult>" \</ListBucketResult>"
expectedListResult = ListObjectsResult True (Just "opaque") [object1] [] expectedListResult = ListObjectsResult True (Just "opaque") [object1] []
object1 = ObjectInfo "my-image.jpg" modifiedTime1 "\"fba9dede5f27731c9771645a39863328\"" 434234 H.empty H.empty object1 = ObjectInfo "my-image.jpg" modifiedTime1 "\"fba9dede5f27731c9771645a39863328\"" 434234 H.empty H.empty
modifiedTime1 = flip UTCTime 64230 $ fromGregorian 2009 10 12 modifiedTime1 = flip UTCTime 64230 $ fromGregorian 2009 10 12
@ -136,8 +135,8 @@ testParseListObjectsResult = do
testParseListObjectsV1Result :: Assertion testParseListObjectsV1Result :: Assertion
testParseListObjectsV1Result = do testParseListObjectsV1Result = do
let let xmldata =
xmldata = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
\<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\ \<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
\<Name>bucket</Name>\ \<Name>bucket</Name>\
\<Prefix/>\ \<Prefix/>\
@ -153,7 +152,6 @@ testParseListObjectsV1Result = do
\<StorageClass>STANDARD</StorageClass>\ \<StorageClass>STANDARD</StorageClass>\
\</Contents>\ \</Contents>\
\</ListBucketResult>" \</ListBucketResult>"
expectedListResult = ListObjectsV1Result True (Just "my-image1.jpg") [object1] [] expectedListResult = ListObjectsV1Result True (Just "my-image1.jpg") [object1] []
object1 = ObjectInfo "my-image.jpg" modifiedTime1 "\"fba9dede5f27731c9771645a39863328\"" 434234 H.empty H.empty object1 = ObjectInfo "my-image.jpg" modifiedTime1 "\"fba9dede5f27731c9771645a39863328\"" 434234 H.empty H.empty
modifiedTime1 = flip UTCTime 64230 $ fromGregorian 2009 10 12 modifiedTime1 = flip UTCTime 64230 $ fromGregorian 2009 10 12
@ -163,8 +161,8 @@ testParseListObjectsV1Result = do
testParseListIncompleteUploads :: Assertion testParseListIncompleteUploads :: Assertion
testParseListIncompleteUploads = do testParseListIncompleteUploads = do
let let xmldata =
xmldata = "<ListMultipartUploadsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\ "<ListMultipartUploadsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
\<Bucket>example-bucket</Bucket>\ \<Bucket>example-bucket</Bucket>\
\<KeyMarker/>\ \<KeyMarker/>\
\<UploadIdMarker/>\ \<UploadIdMarker/>\
@ -203,17 +201,16 @@ testParseListIncompleteUploads = do
parsedListUploadsResult <- tryValidationErr $ runTestNS $ parseListUploadsResponse xmldata parsedListUploadsResult <- tryValidationErr $ runTestNS $ parseListUploadsResponse xmldata
eitherValidationErr parsedListUploadsResult (@?= expectedListResult) eitherValidationErr parsedListUploadsResult (@?= expectedListResult)
testParseCompleteMultipartUploadResponse :: Assertion testParseCompleteMultipartUploadResponse :: Assertion
testParseCompleteMultipartUploadResponse = do testParseCompleteMultipartUploadResponse = do
let let xmldata =
xmldata = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
\<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\ \<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
\<Location>http://Example-Bucket.s3.amazonaws.com/Example-Object</Location>\ \<Location>http://Example-Bucket.s3.amazonaws.com/Example-Object</Location>\
\<Bucket>Example-Bucket</Bucket>\ \<Bucket>Example-Bucket</Bucket>\
\<Key>Example-Object</Key>\ \<Key>Example-Object</Key>\
\<ETag>\"3858f62230ac3c915f300c664312c11f-9\"</ETag>\ \<ETag>\"3858f62230ac3c915f300c664312c11f-9\"</ETag>\
\</CompleteMultipartUploadResult>" \</CompleteMultipartUploadResult>"
expectedETag = "\"3858f62230ac3c915f300c664312c11f-9\"" expectedETag = "\"3858f62230ac3c915f300c664312c11f-9\""
parsedETagE <- runExceptT $ runTestNS $ parseCompleteMultipartUploadResponse xmldata parsedETagE <- runExceptT $ runTestNS $ parseCompleteMultipartUploadResponse xmldata
@ -221,9 +218,9 @@ testParseCompleteMultipartUploadResponse = do
testParseListPartsResponse :: Assertion testParseListPartsResponse :: Assertion
testParseListPartsResponse = do testParseListPartsResponse = do
let let xmldata =
xmldata = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
\<ListPartsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\ \<ListPartsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
\<Bucket>example-bucket</Bucket>\ \<Bucket>example-bucket</Bucket>\
\<Key>example-object</Key>\ \<Key>example-object</Key>\
\<UploadId>XXBsb2FkIElEIGZvciBlbHZpbmcncyVcdS1tb3ZpZS5tMnRzEEEwbG9hZA</UploadId>\ \<UploadId>XXBsb2FkIElEIGZvciBlbHZpbmcncyVcdS1tb3ZpZS5tMnRzEEEwbG9hZA</UploadId>\
@ -252,8 +249,7 @@ testParseListPartsResponse = do
\<ETag>\"aaaa18db4cc2f85cedef654fccc4a4x8\"</ETag>\ \<ETag>\"aaaa18db4cc2f85cedef654fccc4a4x8\"</ETag>\
\<Size>10485760</Size>\ \<Size>10485760</Size>\
\</Part>\ \</Part>\
\</ListPartsResult>" \</ListPartsResult>"
expectedListResult = ListPartsResult True (Just 3) [part1, part2] expectedListResult = ListPartsResult True (Just 3) [part1, part2]
part1 = ObjectPartInfo 2 "\"7778aef83f66abc1fa1e8477f296d394\"" 10485760 modifiedTime1 part1 = ObjectPartInfo 2 "\"7778aef83f66abc1fa1e8477f296d394\"" 10485760 modifiedTime1
modifiedTime1 = flip UTCTime 74914 $ fromGregorian 2010 11 10 modifiedTime1 = flip UTCTime 74914 $ fromGregorian 2010 11 10
@ -265,21 +261,26 @@ testParseListPartsResponse = do
testParseCopyObjectResponse :: Assertion testParseCopyObjectResponse :: Assertion
testParseCopyObjectResponse = do testParseCopyObjectResponse = do
let let cases =
cases = [ ("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\ [ ( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
\<CopyObjectResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\ \<CopyObjectResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
\<LastModified>2009-10-28T22:32:00.000Z</LastModified>\ \<LastModified>2009-10-28T22:32:00.000Z</LastModified>\
\<ETag>\"9b2cf535f27731c974343645a3985328\"</ETag>\ \<ETag>\"9b2cf535f27731c974343645a3985328\"</ETag>\
\</CopyObjectResult>", \</CopyObjectResult>",
("\"9b2cf535f27731c974343645a3985328\"", ( "\"9b2cf535f27731c974343645a3985328\"",
UTCTime (fromGregorian 2009 10 28) 81120)) UTCTime (fromGregorian 2009 10 28) 81120
, ("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\ )
\<CopyPartResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\ ),
( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
\<CopyPartResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
\<LastModified>2009-10-28T22:32:00.000Z</LastModified>\ \<LastModified>2009-10-28T22:32:00.000Z</LastModified>\
\<ETag>\"9b2cf535f27731c974343645a3985328\"</ETag>\ \<ETag>\"9b2cf535f27731c974343645a3985328\"</ETag>\
\</CopyPartResult>", \</CopyPartResult>",
("\"9b2cf535f27731c974343645a3985328\"", ( "\"9b2cf535f27731c974343645a3985328\"",
UTCTime (fromGregorian 2009 10 28) 81120))] UTCTime (fromGregorian 2009 10 28) 81120
)
)
]
forM_ cases $ \(xmldata, (etag, modTime)) -> do forM_ cases $ \(xmldata, (etag, modTime)) -> do
parseResult <- runExceptT $ runTestNS $ parseCopyObjectResponse xmldata parseResult <- runExceptT $ runTestNS $ parseCopyObjectResponse xmldata
@ -287,72 +288,87 @@ testParseCopyObjectResponse = do
testParseNotification :: Assertion testParseNotification :: Assertion
testParseNotification = do testParseNotification = do
let let cases =
cases = [ ("<NotificationConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\ [ ( "<NotificationConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
\ <TopicConfiguration>\ \ <TopicConfiguration>\
\ <Id>YjVkM2Y0YmUtNGI3NC00ZjQyLWEwNGItNDIyYWUxY2I0N2M4</Id>\ \ <Id>YjVkM2Y0YmUtNGI3NC00ZjQyLWEwNGItNDIyYWUxY2I0N2M4</Id>\
\ <Topic>arn:aws:sns:us-east-1:account-id:s3notificationtopic2</Topic>\ \ <Topic>arn:aws:sns:us-east-1:account-id:s3notificationtopic2</Topic>\
\ <Event>s3:ReducedRedundancyLostObject</Event>\ \ <Event>s3:ReducedRedundancyLostObject</Event>\
\ <Event>s3:ObjectCreated:*</Event>\ \ <Event>s3:ObjectCreated:*</Event>\
\ </TopicConfiguration>\ \ </TopicConfiguration>\
\</NotificationConfiguration>", \</NotificationConfiguration>",
Notification [] Notification
[]
[ NotificationConfig [ NotificationConfig
"YjVkM2Y0YmUtNGI3NC00ZjQyLWEwNGItNDIyYWUxY2I0N2M4" "YjVkM2Y0YmUtNGI3NC00ZjQyLWEwNGItNDIyYWUxY2I0N2M4"
"arn:aws:sns:us-east-1:account-id:s3notificationtopic2" "arn:aws:sns:us-east-1:account-id:s3notificationtopic2"
[ReducedRedundancyLostObject, ObjectCreated] defaultFilter [ReducedRedundancyLostObject, ObjectCreated]
defaultFilter
] ]
[]) []
, ("<NotificationConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\ ),
\ <CloudFunctionConfiguration>\ ( "<NotificationConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
\ <Id>ObjectCreatedEvents</Id>\ \ <CloudFunctionConfiguration>\
\ <CloudFunction>arn:aws:lambda:us-west-2:35667example:function:CreateThumbnail</CloudFunction>\ \ <Id>ObjectCreatedEvents</Id>\
\ <Event>s3:ObjectCreated:*</Event>\ \ <CloudFunction>arn:aws:lambda:us-west-2:35667example:function:CreateThumbnail</CloudFunction>\
\ </CloudFunctionConfiguration>\ \ <Event>s3:ObjectCreated:*</Event>\
\ <QueueConfiguration>\ \ </CloudFunctionConfiguration>\
\ <Id>1</Id>\ \ <QueueConfiguration>\
\ <Filter>\ \ <Id>1</Id>\
\ <S3Key>\ \ <Filter>\
\ <FilterRule>\ \ <S3Key>\
\ <Name>prefix</Name>\ \ <FilterRule>\
\ <Value>images/</Value>\ \ <Name>prefix</Name>\
\ </FilterRule>\ \ <Value>images/</Value>\
\ <FilterRule>\ \ </FilterRule>\
\ <Name>suffix</Name>\ \ <FilterRule>\
\ <Value>.jpg</Value>\ \ <Name>suffix</Name>\
\ </FilterRule>\ \ <Value>.jpg</Value>\
\ </S3Key>\ \ </FilterRule>\
\ </Filter>\ \ </S3Key>\
\ <Queue>arn:aws:sqs:us-west-2:444455556666:s3notificationqueue</Queue>\ \ </Filter>\
\ <Event>s3:ObjectCreated:Put</Event>\ \ <Queue>arn:aws:sqs:us-west-2:444455556666:s3notificationqueue</Queue>\
\ </QueueConfiguration>\ \ <Event>s3:ObjectCreated:Put</Event>\
\ <TopicConfiguration>\ \ </QueueConfiguration>\
\ <Topic>arn:aws:sns:us-east-1:356671443308:s3notificationtopic2</Topic>\ \ <TopicConfiguration>\
\ <Event>s3:ReducedRedundancyLostObject</Event>\ \ <Topic>arn:aws:sns:us-east-1:356671443308:s3notificationtopic2</Topic>\
\ </TopicConfiguration>\ \ <Event>s3:ReducedRedundancyLostObject</Event>\
\ <QueueConfiguration>\ \ </TopicConfiguration>\
\ <Queue>arn:aws:sqs:us-east-1:356671443308:s3notificationqueue</Queue>\ \ <QueueConfiguration>\
\ <Event>s3:ObjectCreated:*</Event>\ \ <Queue>arn:aws:sqs:us-east-1:356671443308:s3notificationqueue</Queue>\
\ </QueueConfiguration>)\ \ <Event>s3:ObjectCreated:*</Event>\
\</NotificationConfiguration>", \ </QueueConfiguration>)\
Notification [ NotificationConfig \</NotificationConfiguration>",
"1" "arn:aws:sqs:us-west-2:444455556666:s3notificationqueue" Notification
[ NotificationConfig
"1"
"arn:aws:sqs:us-west-2:444455556666:s3notificationqueue"
[ObjectCreatedPut] [ObjectCreatedPut]
(Filter $ FilterKey $ FilterRules ( Filter $ FilterKey $
[FilterRule "prefix" "images/", FilterRules
FilterRule "suffix" ".jpg"]) [ FilterRule "prefix" "images/",
, NotificationConfig FilterRule "suffix" ".jpg"
"" "arn:aws:sqs:us-east-1:356671443308:s3notificationqueue" ]
[ObjectCreated] defaultFilter ),
NotificationConfig
""
"arn:aws:sqs:us-east-1:356671443308:s3notificationqueue"
[ObjectCreated]
defaultFilter
] ]
[ NotificationConfig [ NotificationConfig
"" "arn:aws:sns:us-east-1:356671443308:s3notificationtopic2" ""
[ReducedRedundancyLostObject] defaultFilter "arn:aws:sns:us-east-1:356671443308:s3notificationtopic2"
[ReducedRedundancyLostObject]
defaultFilter
] ]
[ NotificationConfig [ NotificationConfig
"ObjectCreatedEvents" "arn:aws:lambda:us-west-2:35667example:function:CreateThumbnail" "ObjectCreatedEvents"
[ObjectCreated] defaultFilter "arn:aws:lambda:us-west-2:35667example:function:CreateThumbnail"
]) [ObjectCreated]
defaultFilter
]
)
] ]
forM_ cases $ \(xmldata, val) -> do forM_ cases $ \(xmldata, val) -> do
@ -362,18 +378,23 @@ testParseNotification = do
-- | Tests parsing of both progress and stats -- | Tests parsing of both progress and stats
testParseSelectProgress :: Assertion testParseSelectProgress :: Assertion
testParseSelectProgress = do testParseSelectProgress = do
let cases = [ ([r|<?xml version="1.0" encoding="UTF-8"?> let cases =
[ ( [r|<?xml version="1.0" encoding="UTF-8"?>
<Progress> <Progress>
<BytesScanned>512</BytesScanned> <BytesScanned>512</BytesScanned>
<BytesProcessed>1024</BytesProcessed> <BytesProcessed>1024</BytesProcessed>
<BytesReturned>1024</BytesReturned> <BytesReturned>1024</BytesReturned>
</Progress>|] , Progress 512 1024 1024) </Progress>|],
, ([r|<?xml version="1.0" encoding="UTF-8"?> Progress 512 1024 1024
),
( [r|<?xml version="1.0" encoding="UTF-8"?>
<Stats> <Stats>
<BytesScanned>512</BytesScanned> <BytesScanned>512</BytesScanned>
<BytesProcessed>1024</BytesProcessed> <BytesProcessed>1024</BytesProcessed>
<BytesReturned>1024</BytesReturned> <BytesReturned>1024</BytesReturned>
</Stats>|], Progress 512 1024 1024) </Stats>|],
Progress 512 1024 1024
)
] ]
forM_ cases $ \(xmldata, progress) -> do forM_ cases $ \(xmldata, progress) -> do

View File

@ -14,14 +14,9 @@
-- limitations under the License. -- limitations under the License.
-- --
import Test.Tasty
import Test.Tasty.QuickCheck as QC
import qualified Data.ByteString as B import qualified Data.ByteString as B
import qualified Data.List as L import qualified Data.List as L
import Lib.Prelude import Lib.Prelude
import Network.Minio.API.Test import Network.Minio.API.Test
import Network.Minio.CopyObject import Network.Minio.CopyObject
import Network.Minio.Data import Network.Minio.Data
@ -29,6 +24,8 @@ import Network.Minio.PutObject
import Network.Minio.Utils.Test import Network.Minio.Utils.Test
import Network.Minio.XmlGenerator.Test import Network.Minio.XmlGenerator.Test
import Network.Minio.XmlParser.Test import Network.Minio.XmlParser.Test
import Test.Tasty
import Test.Tasty.QuickCheck as QC
main :: IO () main :: IO ()
main = defaultMain tests main = defaultMain tests
@ -51,70 +48,68 @@ properties = testGroup "Properties" [qcProps] -- [scProps]
-- ] -- ]
qcProps :: TestTree qcProps :: TestTree
qcProps = testGroup "(checked by QuickCheck)" qcProps =
testGroup
"(checked by QuickCheck)"
[ QC.testProperty "selectPartSizes:" $ [ QC.testProperty "selectPartSizes:" $
\n -> let (pns, offs, sizes) = L.unzip3 (selectPartSizes n) \n ->
let (pns, offs, sizes) = L.unzip3 (selectPartSizes n)
-- check that pns increments from 1. -- check that pns increments from 1.
isPNumsAscendingFrom1 = all (\(a, b) -> a == b) $ zip pns [1..] isPNumsAscendingFrom1 = all (\(a, b) -> a == b) $ zip pns [1 ..]
consPairs [] = [] consPairs [] = []
consPairs [_] = [] consPairs [_] = []
consPairs (a:(b:c)) = (a, b):(consPairs (b:c)) consPairs (a : (b : c)) = (a, b) : (consPairs (b : c))
-- check `offs` is monotonically increasing. -- check `offs` is monotonically increasing.
isOffsetsAsc = all (\(a, b) -> a < b) $ consPairs offs isOffsetsAsc = all (\(a, b) -> a < b) $ consPairs offs
-- check sizes sums to n. -- check sizes sums to n.
isSumSizeOk = sum sizes == n isSumSizeOk = sum sizes == n
-- check sizes are constant except last -- check sizes are constant except last
isSizesConstantExceptLast = isSizesConstantExceptLast =
all (\(a, b) -> a == b) (consPairs $ L.init sizes) all (\(a, b) -> a == b) (consPairs $ L.init sizes)
-- check each part except last is at least minPartSize; -- check each part except last is at least minPartSize;
-- last part may be 0 only if it is the only part. -- last part may be 0 only if it is the only part.
nparts = length sizes nparts = length sizes
isMinPartSizeOk = isMinPartSizeOk =
if | nparts > 1 -> -- last part can be smaller but > 0 if
all (>= minPartSize) (take (nparts - 1) sizes) && | nparts > 1 -> -- last part can be smaller but > 0
all (\s -> s > 0) (drop (nparts - 1) sizes) all (>= minPartSize) (take (nparts - 1) sizes)
&& all (\s -> s > 0) (drop (nparts - 1) sizes)
| nparts == 1 -> -- size may be 0 here. | nparts == 1 -> -- size may be 0 here.
maybe True (\x -> x >= 0 && x <= minPartSize) $ maybe True (\x -> x >= 0 && x <= minPartSize) $
headMay sizes headMay sizes
| otherwise -> False | otherwise -> False
in n < 0
in n < 0 || || ( isPNumsAscendingFrom1 && isOffsetsAsc && isSumSizeOk
(isPNumsAscendingFrom1 && isOffsetsAsc && isSumSizeOk && && isSizesConstantExceptLast
isSizesConstantExceptLast && isMinPartSizeOk) && isMinPartSizeOk
),
, QC.testProperty "selectCopyRanges:" $ QC.testProperty "selectCopyRanges:" $
\(start, end) -> \(start, end) ->
let (_, pairs) = L.unzip (selectCopyRanges (start, end)) let (_, pairs) = L.unzip (selectCopyRanges (start, end))
-- is last part's snd offset end? -- is last part's snd offset end?
isLastPartOk = maybe False ((end ==) . snd) $ lastMay pairs isLastPartOk = maybe False ((end ==) . snd) $ lastMay pairs
-- is first part's fst offset start -- is first part's fst offset start
isFirstPartOk = maybe False ((start ==) . fst) $ headMay pairs isFirstPartOk = maybe False ((start ==) . fst) $ headMay pairs
-- each pair is >=64MiB except last, and all those parts -- each pair is >=64MiB except last, and all those parts
-- have same size. -- have same size.
initSizes = maybe [] (map (\(a, b) -> b - a + 1)) $ initMay pairs initSizes = maybe [] (map (\(a, b) -> b - a + 1)) $ initMay pairs
isPartSizesOk = all (>= minPartSize) initSizes && isPartSizesOk =
maybe True (\k -> all (== k) initSizes) all (>= minPartSize) initSizes
&& maybe
True
(\k -> all (== k) initSizes)
(headMay initSizes) (headMay initSizes)
-- returned offsets are contiguous. -- returned offsets are contiguous.
fsts = drop 1 $ map fst pairs fsts = drop 1 $ map fst pairs
snds = take (length pairs - 1) $ map snd pairs snds = take (length pairs - 1) $ map snd pairs
isContParts = length fsts == length snds && isContParts =
and (map (\(a, b) -> a == b + 1) $ zip fsts snds) length fsts == length snds
&& and (map (\(a, b) -> a == b + 1) $ zip fsts snds)
in start < 0 || start > end || in start < 0 || start > end
(isLastPartOk && isFirstPartOk && isPartSizesOk && isContParts) || (isLastPartOk && isFirstPartOk && isPartSizesOk && isContParts),
QC.testProperty "mkSSECKey:" $
, QC.testProperty "mkSSECKey:" $ \w8s ->
\w8s -> let bs = B.pack w8s let bs = B.pack w8s
r = mkSSECKey bs r = mkSSECKey bs
in case r of in case r of
Just _ -> B.length bs == 32 Just _ -> B.length bs == 32
@ -122,11 +117,15 @@ qcProps = testGroup "(checked by QuickCheck)"
] ]
unitTests :: TestTree unitTests :: TestTree
unitTests = testGroup "Unit tests" [ xmlGeneratorTests, xmlParserTests unitTests =
, bucketNameValidityTests testGroup
, objectNameValidityTests "Unit tests"
, parseServerInfoJSONTest [ xmlGeneratorTests,
, parseHealStatusTest xmlParserTests,
, parseHealStartRespTest bucketNameValidityTests,
, limitedMapConcurrentlyTests objectNameValidityTests,
parseServerInfoJSONTest,
parseHealStatusTest,
parseHealStartRespTest,
limitedMapConcurrentlyTests
] ]