Fix XML generation test for S3 SELECT (#161)
- Test was failing because of non-unique ordering of CSV properties. It is fixed by sorting the CSV properties before serialization.
This commit is contained in:
Aditya Manthramurthy
GitHub
parent
5ab80384ae
commit
73bc5b64a0
@ -155,11 +155,13 @@ fromAWSConfigFile = do
|
||||
bool (throwE "FileNotFound") (return ()) fileExists
|
||||
ini <- ExceptT $ Ini.readIniFile awsCredsFile
|
||||
akey <-
|
||||
ExceptT $ return $
|
||||
Ini.lookupValue "default" "aws_access_key_id" ini
|
||||
ExceptT $
|
||||
return $
|
||||
Ini.lookupValue "default" "aws_access_key_id" ini
|
||||
skey <-
|
||||
ExceptT $ return $
|
||||
Ini.lookupValue "default" "aws_secret_access_key" ini
|
||||
ExceptT $
|
||||
return $
|
||||
Ini.lookupValue "default" "aws_secret_access_key" ini
|
||||
return $ Credentials akey skey
|
||||
return $ hush credsE
|
||||
|
||||
@ -856,6 +858,9 @@ instance Monoid CSVProp where
|
||||
mappend (CSVProp a) (CSVProp b) = CSVProp (b <> a)
|
||||
#endif
|
||||
|
||||
csvPropsList :: CSVProp -> [(Text, Text)]
|
||||
csvPropsList (CSVProp h) = sort $ H.toList h
|
||||
|
||||
defaultCSVProp :: CSVProp
|
||||
defaultCSVProp = mempty
|
||||
|
||||
@ -929,10 +934,11 @@ type CSVOutputProp = CSVProp
|
||||
|
||||
-- | quoteFields is an output serialization parameter
|
||||
quoteFields :: QuoteFields -> CSVProp
|
||||
quoteFields q = CSVProp $ H.singleton "QuoteFields" $
|
||||
case q of
|
||||
QuoteFieldsAsNeeded -> "ASNEEDED"
|
||||
QuoteFieldsAlways -> "ALWAYS"
|
||||
quoteFields q = CSVProp $
|
||||
H.singleton "QuoteFields" $
|
||||
case q of
|
||||
QuoteFieldsAsNeeded -> "ASNEEDED"
|
||||
QuoteFieldsAlways -> "ALWAYS"
|
||||
|
||||
-- | Represent the QuoteField setting.
|
||||
data QuoteFields = QuoteFieldsAsNeeded | QuoteFieldsAlways
|
||||
|
||||
@ -23,7 +23,6 @@ module Network.Minio.XmlGenerator
|
||||
where
|
||||
|
||||
import qualified Data.ByteString.Lazy as LBS
|
||||
import qualified Data.HashMap.Strict as H
|
||||
import qualified Data.Text as T
|
||||
import Lib.Prelude
|
||||
import Network.Minio.Data
|
||||
@ -77,8 +76,9 @@ data XNode
|
||||
|
||||
toXML :: Text -> XNode -> ByteString
|
||||
toXML ns node =
|
||||
LBS.toStrict $ renderLBS def $
|
||||
Document (Prologue [] Nothing []) (xmlNode node) []
|
||||
LBS.toStrict $
|
||||
renderLBS def $
|
||||
Document (Prologue [] Nothing []) (xmlNode node) []
|
||||
where
|
||||
xmlNode :: XNode -> Element
|
||||
xmlNode (XNode name nodes) =
|
||||
@ -143,14 +143,14 @@ mkSelectRequest r = LBS.toStrict $ renderLBS def sr
|
||||
[NodeContent $ show $ srExpressionType r]
|
||||
),
|
||||
NodeElement
|
||||
( Element "InputSerialization" mempty
|
||||
$ inputSerializationNodes
|
||||
$ srInputSerialization r
|
||||
( Element "InputSerialization" mempty $
|
||||
inputSerializationNodes $
|
||||
srInputSerialization r
|
||||
),
|
||||
NodeElement
|
||||
( Element "OutputSerialization" mempty
|
||||
$ outputSerializationNodes
|
||||
$ srOutputSerialization r
|
||||
( Element "OutputSerialization" mempty $
|
||||
outputSerializationNodes $
|
||||
srOutputSerialization r
|
||||
)
|
||||
]
|
||||
++ maybe [] reqProgElem (srRequestProgressEnabled r)
|
||||
@ -186,11 +186,11 @@ mkSelectRequest r = LBS.toStrict $ renderLBS def sr
|
||||
]
|
||||
comprTypeNode Nothing = []
|
||||
kvElement (k, v) = Element (Name k Nothing Nothing) mempty [NodeContent v]
|
||||
formatNode (InputFormatCSV (CSVProp h)) =
|
||||
formatNode (InputFormatCSV c) =
|
||||
Element
|
||||
"CSV"
|
||||
mempty
|
||||
(map NodeElement $ map kvElement $ H.toList h)
|
||||
(map NodeElement $ map kvElement $ csvPropsList c)
|
||||
formatNode (InputFormatJSON p) =
|
||||
Element
|
||||
"JSON"
|
||||
@ -208,17 +208,17 @@ mkSelectRequest r = LBS.toStrict $ renderLBS def sr
|
||||
formatNode InputFormatParquet = Element "Parquet" mempty []
|
||||
outputSerializationNodes (OutputSerializationJSON j) =
|
||||
[ NodeElement
|
||||
( Element "JSON" mempty
|
||||
$ rdElem
|
||||
$ jsonopRecordDelimiter j
|
||||
( Element "JSON" mempty $
|
||||
rdElem $
|
||||
jsonopRecordDelimiter j
|
||||
)
|
||||
]
|
||||
outputSerializationNodes (OutputSerializationCSV (CSVProp h)) =
|
||||
outputSerializationNodes (OutputSerializationCSV c) =
|
||||
[ NodeElement $
|
||||
Element
|
||||
"CSV"
|
||||
mempty
|
||||
(map NodeElement $ map kvElement $ H.toList h)
|
||||
(map NodeElement $ map kvElement $ csvPropsList c)
|
||||
]
|
||||
rdElem Nothing = []
|
||||
rdElem (Just t) =
|
||||
|
||||
@ -90,11 +90,12 @@ testMkPutNotificationRequest =
|
||||
"1"
|
||||
"arn:aws:sqs:us-west-2:444455556666:s3notificationqueue"
|
||||
[ObjectCreatedPut]
|
||||
( Filter $ FilterKey $
|
||||
FilterRules
|
||||
[ FilterRule "prefix" "images/",
|
||||
FilterRule "suffix" ".jpg"
|
||||
]
|
||||
( Filter $
|
||||
FilterKey $
|
||||
FilterRules
|
||||
[ FilterRule "prefix" "images/",
|
||||
FilterRule "suffix" ".jpg"
|
||||
]
|
||||
),
|
||||
NotificationConfig
|
||||
""
|
||||
@ -142,32 +143,32 @@ testMkSelectRequest = mapM_ assertFn cases
|
||||
<> quoteEscapeCharacter "\""
|
||||
)
|
||||
(Just False),
|
||||
[r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>GZIP</CompressionType><CSV><QuoteCharacter>"</QuoteCharacter><RecordDelimiter>
|
||||
</RecordDelimiter><FileHeaderInfo>IGNORE</FileHeaderInfo><QuoteEscapeCharacter>"</QuoteEscapeCharacter><FieldDelimiter>,</FieldDelimiter></CSV></InputSerialization><OutputSerialization><CSV><QuoteCharacter>"</QuoteCharacter><QuoteFields>ASNEEDED</QuoteFields><RecordDelimiter>
|
||||
</RecordDelimiter><QuoteEscapeCharacter>"</QuoteEscapeCharacter><FieldDelimiter>,</FieldDelimiter></CSV></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|]
|
||||
[r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>GZIP</CompressionType><CSV><FieldDelimiter>,</FieldDelimiter><FileHeaderInfo>IGNORE</FileHeaderInfo><QuoteCharacter>"</QuoteCharacter><QuoteEscapeCharacter>"</QuoteEscapeCharacter><RecordDelimiter>
|
||||
</RecordDelimiter></CSV></InputSerialization><OutputSerialization><CSV><FieldDelimiter>,</FieldDelimiter><QuoteCharacter>"</QuoteCharacter><QuoteEscapeCharacter>"</QuoteEscapeCharacter><QuoteFields>ASNEEDED</QuoteFields><RecordDelimiter>
|
||||
</RecordDelimiter></CSV></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|]
|
||||
),
|
||||
( setRequestProgressEnabled False
|
||||
$ setInputCompressionType CompressionTypeGzip
|
||||
$ selectRequest
|
||||
"Select * from S3Object"
|
||||
documentJsonInput
|
||||
(outputJSONFromRecordDelimiter "\n"),
|
||||
( setRequestProgressEnabled False $
|
||||
setInputCompressionType CompressionTypeGzip $
|
||||
selectRequest
|
||||
"Select * from S3Object"
|
||||
documentJsonInput
|
||||
(outputJSONFromRecordDelimiter "\n"),
|
||||
[r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>GZIP</CompressionType><JSON><Type>DOCUMENT</Type></JSON></InputSerialization><OutputSerialization><JSON><RecordDelimiter>
|
||||
</RecordDelimiter></JSON></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|]
|
||||
),
|
||||
( setRequestProgressEnabled False
|
||||
$ setInputCompressionType CompressionTypeNone
|
||||
$ selectRequest
|
||||
"Select * from S3Object"
|
||||
defaultParquetInput
|
||||
( outputCSVFromProps $
|
||||
quoteFields QuoteFieldsAsNeeded
|
||||
<> recordDelimiter "\n"
|
||||
<> fieldDelimiter ","
|
||||
<> quoteCharacter "\""
|
||||
<> quoteEscapeCharacter "\""
|
||||
),
|
||||
[r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>NONE</CompressionType><Parquet/></InputSerialization><OutputSerialization><CSV><QuoteCharacter>"</QuoteCharacter><QuoteFields>ASNEEDED</QuoteFields><RecordDelimiter>
|
||||
</RecordDelimiter><QuoteEscapeCharacter>"</QuoteEscapeCharacter><FieldDelimiter>,</FieldDelimiter></CSV></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|]
|
||||
( setRequestProgressEnabled False $
|
||||
setInputCompressionType CompressionTypeNone $
|
||||
selectRequest
|
||||
"Select * from S3Object"
|
||||
defaultParquetInput
|
||||
( outputCSVFromProps $
|
||||
quoteFields QuoteFieldsAsNeeded
|
||||
<> recordDelimiter "\n"
|
||||
<> fieldDelimiter ","
|
||||
<> quoteCharacter "\""
|
||||
<> quoteEscapeCharacter "\""
|
||||
),
|
||||
[r|<?xml version="1.0" encoding="UTF-8"?><SelectRequest><Expression>Select * from S3Object</Expression><ExpressionType>SQL</ExpressionType><InputSerialization><CompressionType>NONE</CompressionType><Parquet/></InputSerialization><OutputSerialization><CSV><FieldDelimiter>,</FieldDelimiter><QuoteCharacter>"</QuoteCharacter><QuoteEscapeCharacter>"</QuoteEscapeCharacter><QuoteFields>ASNEEDED</QuoteFields><RecordDelimiter>
|
||||
</RecordDelimiter></CSV></OutputSerialization><RequestProgress><Enabled>FALSE</Enabled></RequestProgress></SelectRequest>|]
|
||||
)
|
||||
]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user