haskell/minio-hs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix XML generation test for S3 SELECT (#161)

Browse Source 
- Test was failing because of non-unique ordering of CSV properties. It is fixed
by sorting the CSV properties before serialization.
This commit is contained in:
Aditya Manthramurthy 2021-03-03 15:44:12 -08:00 committed by GitHub
parent 5ab80384ae
commit 73bc5b64a0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 59 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/Network/Minio/Data.hs
View File

@ -155,11 +155,13 @@ fromAWSConfigFile = do
bool (throwE "FileNotFound") (return ()) fileExists bool (throwE "FileNotFound") (return ()) fileExists
ini <- ExceptT $ Ini.readIniFile awsCredsFile ini <- ExceptT $ Ini.readIniFile awsCredsFile
akey <- akey <-
ExceptT $ return $ ExceptT $
Ini.lookupValue "default" "aws_access_key_id" ini return $
Ini.lookupValue "default" "aws_access_key_id" ini
skey <- skey <-
ExceptT $ return $ ExceptT $
Ini.lookupValue "default" "aws_secret_access_key" ini return $
Ini.lookupValue "default" "aws_secret_access_key" ini
return $ Credentials akey skey return $ Credentials akey skey
return $ hush credsE return $ hush credsE
@ -856,6 +858,9 @@ instance Monoid CSVProp where
mappend (CSVProp a) (CSVProp b) = CSVProp (b <> a) mappend (CSVProp a) (CSVProp b) = CSVProp (b <> a)
#endif #endif
csvPropsList :: CSVProp -> [(Text, Text)]
csvPropsList (CSVProp h) = sort $ H.toList h
defaultCSVProp :: CSVProp defaultCSVProp :: CSVProp
defaultCSVProp = mempty defaultCSVProp = mempty
@ -929,10 +934,11 @@ type CSVOutputProp = CSVProp
-- | quoteFields is an output serialization parameter -- | quoteFields is an output serialization parameter
quoteFields :: QuoteFields -> CSVProp quoteFields :: QuoteFields -> CSVProp
quoteFields q = CSVProp $ H.singleton "QuoteFields" $ quoteFields q = CSVProp $
case q of H.singleton "QuoteFields" $
QuoteFieldsAsNeeded -> "ASNEEDED" case q of
QuoteFieldsAlways -> "ALWAYS" QuoteFieldsAsNeeded -> "ASNEEDED"
QuoteFieldsAlways -> "ALWAYS"
-- | Represent the QuoteField setting. -- | Represent the QuoteField setting.
data QuoteFields = QuoteFieldsAsNeeded | QuoteFieldsAlways data QuoteFields = QuoteFieldsAsNeeded | QuoteFieldsAlways

32
src/Network/Minio/XmlGenerator.hs
View File

@ -23,7 +23,6 @@ module Network.Minio.XmlGenerator
where where
import qualified Data.ByteString.Lazy as LBS import qualified Data.ByteString.Lazy as LBS
import qualified Data.HashMap.Strict as H
import qualified Data.Text as T import qualified Data.Text as T
import Lib.Prelude import Lib.Prelude
import Network.Minio.Data import Network.Minio.Data
@ -77,8 +76,9 @@ data XNode
toXML :: Text -> XNode -> ByteString toXML :: Text -> XNode -> ByteString
toXML ns node = toXML ns node =
LBS.toStrict $ renderLBS def $ LBS.toStrict $
Document (Prologue [] Nothing []) (xmlNode node) [] renderLBS def $
Document (Prologue [] Nothing []) (xmlNode node) []
where where
xmlNode :: XNode -> Element xmlNode :: XNode -> Element
xmlNode (XNode name nodes) = xmlNode (XNode name nodes) =
@ -143,14 +143,14 @@ mkSelectRequest r = LBS.toStrict $ renderLBS def sr
[NodeContent $ show $ srExpressionType r] [NodeContent $ show $ srExpressionType r]
), ),
NodeElement NodeElement
( Element "InputSerialization" mempty ( Element "InputSerialization" mempty $
$ inputSerializationNodes inputSerializationNodes $
$ srInputSerialization r srInputSerialization r
), ),
NodeElement NodeElement
( Element "OutputSerialization" mempty ( Element "OutputSerialization" mempty $
$ outputSerializationNodes outputSerializationNodes $
$ srOutputSerialization r srOutputSerialization r
) )
] ]
++ maybe [] reqProgElem (srRequestProgressEnabled r) ++ maybe [] reqProgElem (srRequestProgressEnabled r)
@ -186,11 +186,11 @@ mkSelectRequest r = LBS.toStrict $ renderLBS def sr
] ]
comprTypeNode Nothing = [] comprTypeNode Nothing = []
kvElement (k, v) = Element (Name k Nothing Nothing) mempty [NodeContent v] kvElement (k, v) = Element (Name k Nothing Nothing) mempty [NodeContent v]
formatNode (InputFormatCSV (CSVProp h)) = formatNode (InputFormatCSV c) =
Element Element
"CSV" "CSV"
mempty mempty
(map NodeElement $ map kvElement $ H.toList h) (map NodeElement $ map kvElement $ csvPropsList c)
formatNode (InputFormatJSON p) = formatNode (InputFormatJSON p) =
Element Element
"JSON" "JSON"
@ -208,17 +208,17 @@ mkSelectRequest r = LBS.toStrict $ renderLBS def sr
formatNode InputFormatParquet = Element "Parquet" mempty [] formatNode InputFormatParquet = Element "Parquet" mempty []
outputSerializationNodes (OutputSerializationJSON j) = outputSerializationNodes (OutputSerializationJSON j) =
[ NodeElement [ NodeElement
( Element "JSON" mempty ( Element "JSON" mempty $
$ rdElem rdElem $
$ jsonopRecordDelimiter j jsonopRecordDelimiter j
) )
] ]
outputSerializationNodes (OutputSerializationCSV (CSVProp h)) = outputSerializationNodes (OutputSerializationCSV c) =
[ NodeElement $ [ NodeElement $
Element Element
"CSV" "CSV"
mempty mempty
(map NodeElement $ map kvElement $ H.toList h) (map NodeElement $ map kvElement $ csvPropsList c)
] ]
rdElem Nothing = [] rdElem Nothing = []
rdElem (Just t) = rdElem (Just t) =

57
test/Network/Minio/XmlGenerator/Test.hs
View File

@ -90,11 +90,12 @@ testMkPutNotificationRequest =
"1" "1"
"arn:aws:sqs:us-west-2:444455556666:s3notificationqueue" "arn:aws:sqs:us-west-2:444455556666:s3notificationqueue"
[ObjectCreatedPut] [ObjectCreatedPut]
( Filter $ FilterKey $ ( Filter $
FilterRules FilterKey $
[ FilterRule "prefix" "images/", FilterRules
FilterRule "suffix" ".jpg" [ FilterRule "prefix" "images/",
] FilterRule "suffix" ".jpg"
]
), ),
NotificationConfig NotificationConfig
"" ""
@ -142,32 +143,32 @@ testMkSelectRequest = mapM_ assertFn cases
<> quoteEscapeCharacter "\"" <> quoteEscapeCharacter "\""
) )
(Just False), (Just False),
[r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>GZIP</CompressionType><CSV><QuoteCharacter>&#34;</QuoteCharacter><RecordDelimiter> [r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>GZIP</CompressionType><CSV><FieldDelimiter>,</FieldDelimiter><FileHeaderInfo>IGNORE</FileHeaderInfo><QuoteCharacter>&#34;</QuoteCharacter><QuoteEscapeCharacter>&#34;</QuoteEscapeCharacter><RecordDelimiter>
</RecordDelimiter><FileHeaderInfo>IGNORE</FileHeaderInfo><QuoteEscapeCharacter>&#34;</QuoteEscapeCharacter><FieldDelimiter>,</FieldDelimiter></CSV></InputSerialization><OutputSerialization><CSV><QuoteCharacter>&#34;</QuoteCharacter><QuoteFields>ASNEEDED</QuoteFields><RecordDelimiter> </RecordDelimiter></CSV></InputSerialization><OutputSerialization><CSV><FieldDelimiter>,</FieldDelimiter><QuoteCharacter>&#34;</QuoteCharacter><QuoteEscapeCharacter>&#34;</QuoteEscapeCharacter><QuoteFields>ASNEEDED</QuoteFields><RecordDelimiter>
</RecordDelimiter><QuoteEscapeCharacter>&#34;</QuoteEscapeCharacter><FieldDelimiter>,</FieldDelimiter></CSV></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|] </RecordDelimiter></CSV></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|]
), ),
( setRequestProgressEnabled False ( setRequestProgressEnabled False $
$ setInputCompressionType CompressionTypeGzip setInputCompressionType CompressionTypeGzip $
$ selectRequest selectRequest
"Select * from S3Object" "Select * from S3Object"
documentJsonInput documentJsonInput
(outputJSONFromRecordDelimiter "\n"), (outputJSONFromRecordDelimiter "\n"),
[r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>GZIP</CompressionType><JSON><Type>DOCUMENT</Type></JSON></InputSerialization><OutputSerialization><JSON><RecordDelimiter> [r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>GZIP</CompressionType><JSON><Type>DOCUMENT</Type></JSON></InputSerialization><OutputSerialization><JSON><RecordDelimiter>
</RecordDelimiter></JSON></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|] </RecordDelimiter></JSON></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|]
), ),
( setRequestProgressEnabled False ( setRequestProgressEnabled False $
$ setInputCompressionType CompressionTypeNone setInputCompressionType CompressionTypeNone $
$ selectRequest selectRequest
"Select * from S3Object" "Select * from S3Object"
defaultParquetInput defaultParquetInput
( outputCSVFromProps $ ( outputCSVFromProps $
quoteFields QuoteFieldsAsNeeded quoteFields QuoteFieldsAsNeeded
<> recordDelimiter "\n" <> recordDelimiter "\n"
<> fieldDelimiter "," <> fieldDelimiter ","
<> quoteCharacter "\"" <> quoteCharacter "\""
<> quoteEscapeCharacter "\"" <> quoteEscapeCharacter "\""
), ),
[r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>NONE</CompressionType><Parquet/></InputSerialization><OutputSerialization><CSV><QuoteCharacter>&#34;</QuoteCharacter><QuoteFields>ASNEEDED</QuoteFields><RecordDelimiter> [r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>NONE</CompressionType><Parquet/></InputSerialization><OutputSerialization><CSV><FieldDelimiter>,</FieldDelimiter><QuoteCharacter>&#34;</QuoteCharacter><QuoteEscapeCharacter>&#34;</QuoteEscapeCharacter><QuoteFields>ASNEEDED</QuoteFields><RecordDelimiter>
</RecordDelimiter><QuoteEscapeCharacter>&#34;</QuoteEscapeCharacter><FieldDelimiter>,</FieldDelimiter></CSV></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|] </RecordDelimiter></CSV></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|]
) )
] ]