Add putObjectOptions for PutObject (#71)

This commit is contained in:
Harshavardhana 2018-01-16 00:59:17 -08:00 committed by Krishnan Parthasarathi
parent 8be1ff429f
commit 37940ad170
11 changed files with 160 additions and 55 deletions

View File

@ -47,11 +47,12 @@ main = do
res <- runMinio minioPlayCI $ do res <- runMinio minioPlayCI $ do
liftIO $ B.putStrLn "Upload a file that we will fetch with a presigned URL..." liftIO $ B.putStrLn "Upload a file that we will fetch with a presigned URL..."
putObject bucket object (CC.repeat "a") (Just kb15) putObject bucket object (CC.repeat "a") (Just kb15) def
liftIO $ putStrLn $ "Done. Object created at: my-bucket/my-object" liftIO $ putStrLn $ "Done. Object created at: my-bucket/my-object"
-- Extract Etag of uploaded object -- Extract Etag of uploaded object
(ObjectInfo _ _ etag _) <- statObject bucket object oi <- statObject bucket object
let etag = oiETag oi
-- Set header to add an if-match constraint - this makes sure -- Set header to add an if-match constraint - this makes sure
-- the fetching fails if the object is changed on the server -- the fetching fails if the object is changed on the server

View File

@ -37,17 +37,17 @@ main = do
localFile = "/etc/lsb-release" localFile = "/etc/lsb-release"
kb15 = 15 * 1024 kb15 = 15 * 1024
-- Eg 1. Upload a stream of repeating "a" using putObject. -- Eg 1. Upload a stream of repeating "a" using putObject with default options.
res1 <- runMinio minioPlayCI $ res1 <- runMinio minioPlayCI $
putObject bucket object (CC.repeat "a") (Just kb15) putObject bucket object (CC.repeat "a") (Just kb15) def
case res1 of case res1 of
Left e -> putStrLn $ "putObject failed." ++ show e Left e -> putStrLn $ "putObject failed." ++ show e
Right () -> putStrLn "putObject succeeded." Right () -> putStrLn "putObject succeeded."
-- Eg 2. Upload a file using fPutObject. -- Eg 2. Upload a file using fPutObject with default options.
res2 <- runMinio minioPlayCI $ res2 <- runMinio minioPlayCI $
fPutObject bucket object localFile fPutObject bucket object localFile def
case res2 of case res2 of
Left e -> putStrLn $ "fPutObject failed." ++ show e Left e -> putStrLn $ "fPutObject failed." ++ show e
Right () -> putStrLn "fPutObject succeeded." Right () -> putStrLn "fPutObject succeeded."

View File

@ -55,9 +55,18 @@ module Network.Minio
-- ** Listing -- ** Listing
, BucketInfo(..) , BucketInfo(..)
, listBuckets , listBuckets
, ObjectInfo(..)
-- ** Object info type represents object metadata information.
, ObjectInfo
, oiObject
, oiModTime
, oiETag
, oiSize
, oiMetadata
, listObjects , listObjects
, listObjectsV1 , listObjectsV1
, UploadId , UploadId
, UploadInfo(..) , UploadInfo(..)
, listIncompleteUploads , listIncompleteUploads
@ -88,6 +97,15 @@ module Network.Minio
-- ** Conduit-based streaming operations -- ** Conduit-based streaming operations
, putObject , putObject
-- | Input data type represents PutObject options.
, PutObjectOptions
, pooContentType
, pooContentEncoding
, pooContentDisposition
, pooCacheControl
, pooUserMetadata
, pooNumThreads
, getObject , getObject
-- | Input data type represents GetObject options. -- | Input data type represents GetObject options.
, GetObjectOptions , GetObjectOptions
@ -183,18 +201,19 @@ fGetObject bucket object fp opts = do
src C.$$+- CB.sinkFileCautious fp src C.$$+- CB.sinkFileCautious fp
-- | Upload the given file to the given object. -- | Upload the given file to the given object.
fPutObject :: Bucket -> Object -> FilePath -> Minio () fPutObject :: Bucket -> Object -> FilePath
fPutObject bucket object f = void $ putObjectInternal bucket object $ -> PutObjectOptions -> Minio ()
ODFile f Nothing fPutObject bucket object f opts =
void $ putObjectInternal bucket object opts $ ODFile f Nothing
-- | Put an object from a conduit source. The size can be provided if -- | Put an object from a conduit source. The size can be provided if
-- known; this helps the library select optimal part sizes to perform -- known; this helps the library select optimal part sizes to perform
-- a multipart upload. If not specified, it is assumed that the object -- a multipart upload. If not specified, it is assumed that the object
-- can be potentially 5TiB and selects multipart sizes appropriately. -- can be potentially 5TiB and selects multipart sizes appropriately.
putObject :: Bucket -> Object -> C.Producer Minio ByteString putObject :: Bucket -> Object -> C.Producer Minio ByteString
-> Maybe Int64 -> Minio () -> Maybe Int64 -> PutObjectOptions -> Minio ()
putObject bucket object src sizeMay = putObject bucket object src sizeMay opts =
void $ putObjectInternal bucket object $ ODStream src sizeMay void $ putObjectInternal bucket object opts $ ODStream src sizeMay
-- | Perform a server-side copy operation to create an object based on -- | Perform a server-side copy operation to create an object based on
-- the destination specification in DestinationInfo from the source -- the destination specification in DestinationInfo from the source

View File

@ -35,7 +35,8 @@ copyObjectInternal b' o srcInfo = do
sObject = srcObject srcInfo sObject = srcObject srcInfo
-- get source object size with a head request -- get source object size with a head request
(ObjectInfo _ _ _ srcSize) <- headObject sBucket sObject oi <- headObject sBucket sObject
let srcSize = oiSize oi
-- check that byte offsets are valid if specified in cps -- check that byte offsets are valid if specified in cps
let rangeMay = srcRange srcInfo let rangeMay = srcRange srcInfo

View File

@ -27,6 +27,7 @@ import qualified Data.ByteString as B
import Data.Default (Default (..)) import Data.Default (Default (..))
import qualified Data.Map as Map import qualified Data.Map as Map
import qualified Data.Text as T import qualified Data.Text as T
import Data.CaseInsensitive (mk)
import Data.Time (defaultTimeLocale, formatTime) import Data.Time (defaultTimeLocale, formatTime)
import Network.HTTP.Client (defaultManagerSettings) import Network.HTTP.Client (defaultManagerSettings)
import qualified Network.HTTP.Conduit as NC import qualified Network.HTTP.Conduit as NC
@ -39,7 +40,6 @@ import GHC.Show (Show (..))
import Lib.Prelude import Lib.Prelude
-- | max obj size is 5TiB -- | max obj size is 5TiB
maxObjectSize :: Int64 maxObjectSize :: Int64
maxObjectSize = 5 * 1024 * 1024 * oneMiB maxObjectSize = 5 * 1024 * 1024 * oneMiB
@ -182,6 +182,45 @@ type Region = Text
-- APIs. -- APIs.
type ETag = Text type ETag = Text
-- |
-- Data type represents various options specified for PutObject call.
-- To specify PutObject options use the poo* accessors.
data PutObjectOptions = PutObjectOptions {
pooContentType :: Maybe Text
, pooContentEncoding :: Maybe Text
, pooContentDisposition :: Maybe Text
, pooCacheControl :: Maybe Text
, pooUserMetadata :: [(Text, Text)]
, pooNumThreads :: Maybe Word
} deriving (Show, Eq)
-- Provide a default instance
instance Default PutObjectOptions where
def = PutObjectOptions def def def def [] def
addXAmzMetaPrefix :: Text -> Text
addXAmzMetaPrefix s = do
if (T.isPrefixOf "x-amz-meta-" s)
then s
else T.concat ["x-amz-meta-", s]
mkHeaderFromMetadata :: [(Text, Text)] -> [HT.Header]
mkHeaderFromMetadata = map (\(x, y) -> (mk $ encodeUtf8 $ addXAmzMetaPrefix $ T.toLower x, encodeUtf8 y))
pooToHeaders :: PutObjectOptions -> [HT.Header]
pooToHeaders poo = userMetadata ++ zip names values
where
userMetadata = mkHeaderFromMetadata $ pooUserMetadata poo
names = ["content-type",
"content-encoding",
"content-disposition",
"cache-control"]
values = mapMaybe (fmap encodeUtf8 . (poo &))
[pooContentType, pooContentEncoding,
pooContentDisposition, pooCacheControl]
-- | -- |
-- BucketInfo returned for list buckets call -- BucketInfo returned for list buckets call
data BucketInfo = BucketInfo { data BucketInfo = BucketInfo {
@ -206,7 +245,6 @@ data ListPartsResult = ListPartsResult {
, lprParts :: [ObjectPartInfo] , lprParts :: [ObjectPartInfo]
} deriving (Show, Eq) } deriving (Show, Eq)
-- | Represents information about an object part in an ongoing -- | Represents information about an object part in an ongoing
-- multipart upload. -- multipart upload.
data ObjectPartInfo = ObjectPartInfo { data ObjectPartInfo = ObjectPartInfo {
@ -256,6 +294,7 @@ data ObjectInfo = ObjectInfo {
, oiModTime :: UTCTime , oiModTime :: UTCTime
, oiETag :: ETag , oiETag :: ETag
, oiSize :: Int64 , oiSize :: Int64
, oiMetadata :: Map.Map Text Text
} deriving (Show, Eq) } deriving (Show, Eq)
-- | Represents source object in server-side copy object -- | Represents source object in server-side copy object
@ -267,7 +306,7 @@ data SourceInfo = SourceInfo {
, srcIfNoneMatch :: Maybe Text , srcIfNoneMatch :: Maybe Text
, srcIfModifiedSince :: Maybe UTCTime , srcIfModifiedSince :: Maybe UTCTime
, srcIfUnmodifiedSince :: Maybe UTCTime , srcIfUnmodifiedSince :: Maybe UTCTime
} deriving (Show, Eq) } deriving (Show, Eq)
instance Default SourceInfo where instance Default SourceInfo where
def = SourceInfo "" "" def def def def def def = SourceInfo "" "" def def def def def

View File

@ -52,9 +52,10 @@ data ObjectData m =
-- | Put an object from ObjectData. This high-level API handles -- | Put an object from ObjectData. This high-level API handles
-- objects of all sizes, and even if the object size is unknown. -- objects of all sizes, and even if the object size is unknown.
putObjectInternal :: Bucket -> Object -> ObjectData Minio -> Minio ETag putObjectInternal :: Bucket -> Object -> PutObjectOptions
putObjectInternal b o (ODStream src sizeMay) = sequentialMultipartUpload b o sizeMay src -> ObjectData Minio -> Minio ETag
putObjectInternal b o (ODFile fp sizeMay) = do putObjectInternal b o opts (ODStream src sizeMay) = sequentialMultipartUpload b o opts sizeMay src
putObjectInternal b o opts (ODFile fp sizeMay) = do
hResE <- withNewHandle fp $ \h -> hResE <- withNewHandle fp $ \h ->
liftM2 (,) (isHandleSeekable h) (getFileSize h) liftM2 (,) (isHandleSeekable h) (getFileSize h)
@ -66,28 +67,30 @@ putObjectInternal b o (ODFile fp sizeMay) = do
case finalSizeMay of case finalSizeMay of
-- unable to get size, so assume non-seekable file and max-object size -- unable to get size, so assume non-seekable file and max-object size
Nothing -> sequentialMultipartUpload b o (Just maxObjectSize) $ Nothing -> sequentialMultipartUpload b o opts (Just maxObjectSize) $
CB.sourceFile fp CB.sourceFile fp
-- got file size, so check for single/multipart upload -- got file size, so check for single/multipart upload
Just size -> Just size ->
if | size <= 64 * oneMiB -> either throwM return =<< if | size <= 64 * oneMiB -> either throwM return =<<
withNewHandle fp (\h -> putObjectSingle b o [] h 0 size) withNewHandle fp (\h -> putObjectSingle b o (pooToHeaders opts) h 0 size)
| size > maxObjectSize -> throwM $ MErrVPutSizeExceeded size | size > maxObjectSize -> throwM $ MErrVPutSizeExceeded size
| isSeekable -> parallelMultipartUpload b o fp size | isSeekable -> parallelMultipartUpload b o opts fp size
| otherwise -> sequentialMultipartUpload b o (Just size) $ | otherwise -> sequentialMultipartUpload b o opts (Just size) $
CB.sourceFile fp CB.sourceFile fp
parallelMultipartUpload :: Bucket -> Object -> FilePath -> Int64 parallelMultipartUpload :: Bucket -> Object -> PutObjectOptions
-> Minio ETag -> FilePath -> Int64 -> Minio ETag
parallelMultipartUpload b o filePath size = do parallelMultipartUpload b o opts filePath size = do
-- get a new upload id. -- get a new upload id.
uploadId <- newMultipartUpload b o [] uploadId <- newMultipartUpload b o (pooToHeaders opts)
let partSizeInfo = selectPartSizes size let partSizeInfo = selectPartSizes size
-- perform upload with 10 threads let threads = fromMaybe 10 $ pooNumThreads opts
uploadedPartsE <- limitedMapConcurrently 10
-- perform upload with 'threads' threads
uploadedPartsE <- limitedMapConcurrently (fromIntegral threads)
(uploadPart uploadId) partSizeInfo (uploadPart uploadId) partSizeInfo
-- if there were any errors, rethrow exception. -- if there were any errors, rethrow exception.
@ -95,6 +98,7 @@ parallelMultipartUpload b o filePath size = do
-- if we get here, all parts were successfully uploaded. -- if we get here, all parts were successfully uploaded.
completeMultipartUpload b o uploadId $ rights uploadedPartsE completeMultipartUpload b o uploadId $ rights uploadedPartsE
where where
uploadPart uploadId (partNum, offset, sz) = uploadPart uploadId (partNum, offset, sz) =
withNewHandle filePath $ \h -> do withNewHandle filePath $ \h -> do
@ -102,11 +106,13 @@ parallelMultipartUpload b o filePath size = do
putObjectPart b o uploadId partNum [] payload putObjectPart b o uploadId partNum [] payload
-- | Upload multipart object from conduit source sequentially -- | Upload multipart object from conduit source sequentially
sequentialMultipartUpload :: Bucket -> Object -> Maybe Int64 sequentialMultipartUpload :: Bucket -> Object -> PutObjectOptions
-> C.Producer Minio ByteString -> Minio ETag -> Maybe Int64
sequentialMultipartUpload b o sizeMay src = do -> C.Producer Minio ByteString
-> Minio ETag
sequentialMultipartUpload b o opts sizeMay src = do
-- get a new upload id. -- get a new upload id.
uploadId <- newMultipartUpload b o [] uploadId <- newMultipartUpload b o (pooToHeaders opts)
-- upload parts in loop -- upload parts in loop
let partSizes = selectPartSizes $ maybe maxObjectSize identity sizeMay let partSizes = selectPartSizes $ maybe maxObjectSize identity sizeMay

View File

@ -382,10 +382,10 @@ headObject bucket object = do
modTime = getLastModifiedHeader headers modTime = getLastModifiedHeader headers
etag = getETagHeader headers etag = getETagHeader headers
size = getContentLength headers size = getContentLength headers
metadata = getMetadataMap headers
maybe (throwM MErrVInvalidObjectInfoResponse) return $ maybe (throwM MErrVInvalidObjectInfoResponse) return $
ObjectInfo <$> Just object <*> modTime <*> etag <*> size ObjectInfo <$> Just object <*> modTime <*> etag <*> size <*> Just metadata
-- | Query the object store if a given bucket exists. -- | Query the object store if a given bucket exists.

View File

@ -22,9 +22,11 @@ import qualified Control.Exception.Lifted as ExL
import qualified Control.Monad.Catch as MC import qualified Control.Monad.Catch as MC
import qualified Control.Monad.Trans.Resource as R import qualified Control.Monad.Trans.Resource as R
import qualified Data.Map as Map
import qualified Data.ByteString as B import qualified Data.ByteString as B
import qualified Data.ByteString.Lazy as LB import qualified Data.ByteString.Lazy as LB
import Data.CaseInsensitive (mk) import Data.CaseInsensitive (mk)
import Data.CaseInsensitive (original)
import qualified Data.Conduit as C import qualified Data.Conduit as C
import qualified Data.Conduit.Binary as CB import qualified Data.Conduit.Binary as CB
import qualified Data.List as List import qualified Data.List as List
@ -42,6 +44,7 @@ import qualified System.IO as IO
import Lib.Prelude import Lib.Prelude
import Network.Minio.Data import Network.Minio.Data
import Network.Minio.Data.ByteString
import Network.Minio.XmlParser (parseErrResponse) import Network.Minio.XmlParser (parseErrResponse)
allocateReadFile :: (R.MonadResource m, R.MonadResourceBase m, MonadCatch m) allocateReadFile :: (R.MonadResource m, R.MonadResourceBase m, MonadCatch m)
@ -100,6 +103,12 @@ lookupHeader hdr = headMay . map snd . filter (\(h, _) -> h == hdr)
getETagHeader :: [HT.Header] -> Maybe Text getETagHeader :: [HT.Header] -> Maybe Text
getETagHeader hs = decodeUtf8Lenient <$> lookupHeader Hdr.hETag hs getETagHeader hs = decodeUtf8Lenient <$> lookupHeader Hdr.hETag hs
getMetadata :: [HT.Header] -> [(Text, Text)]
getMetadata = map ((\(x, y) -> (decodeUtf8Lenient $ original x, decodeUtf8Lenient $ stripBS y)))
getMetadataMap :: [HT.Header] -> Map Text Text
getMetadataMap hs = Map.fromList (getMetadata hs)
getLastModifiedHeader :: [HT.Header] -> Maybe UTCTime getLastModifiedHeader :: [HT.Header] -> Maybe UTCTime
getLastModifiedHeader hs = do getLastModifiedHeader hs = do
modTimebs <- decodeUtf8Lenient <$> lookupHeader Hdr.hLastModified hs modTimebs <- decodeUtf8Lenient <$> lookupHeader Hdr.hLastModified hs

View File

@ -29,8 +29,9 @@ module Network.Minio.XmlParser
) where ) where
import Control.Monad.Trans.Resource import Control.Monad.Trans.Resource
import Data.List (zip3, zip4) import Data.List (zip3, zip4, zip5)
import qualified Data.Text as T import qualified Data.Text as T
import qualified Data.Map as Map
import Data.Text.Read (decimal) import Data.Text.Read (decimal)
import Data.Time import Data.Time
import Text.XML import Text.XML
@ -50,6 +51,9 @@ s3TimeFormat = iso8601DateFormat $ Just "%T%QZ"
uncurry4 :: (a -> b -> c -> d -> e) -> (a, b, c, d) -> e uncurry4 :: (a -> b -> c -> d -> e) -> (a, b, c, d) -> e
uncurry4 f (a, b, c, d) = f a b c d uncurry4 f (a, b, c, d) = f a b c d
uncurry5 :: (a -> b -> c -> d -> e -> f) -> (a, b, c, d, e) -> f
uncurry5 f (a, b, c, d, e) = f a b c d e
-- | Parse time strings from XML -- | Parse time strings from XML
parseS3XMLTime :: (MonadThrow m) => Text -> m UTCTime parseS3XMLTime :: (MonadThrow m) => Text -> m UTCTime
parseS3XMLTime = either (throwM . MErrVXmlParse) return parseS3XMLTime = either (throwM . MErrVXmlParse) return
@ -134,7 +138,7 @@ parseListObjectsV1Response xmldata = do
sizes <- parseDecimals sizeStr sizes <- parseDecimals sizeStr
let let
objects = map (uncurry4 ObjectInfo) $ zip4 keys modTimes etags sizes objects = map (uncurry5 ObjectInfo) $ zip5 keys modTimes etags sizes (repeat Map.empty)
return $ ListObjectsV1Result hasMore nextMarker objects prefixes return $ ListObjectsV1Result hasMore nextMarker objects prefixes
@ -161,7 +165,7 @@ parseListObjectsResponse xmldata = do
sizes <- parseDecimals sizeStr sizes <- parseDecimals sizeStr
let let
objects = map (uncurry4 ObjectInfo) $ zip4 keys modTimes etags sizes objects = map (uncurry5 ObjectInfo) $ zip5 keys modTimes etags sizes (repeat Map.empty)
return $ ListObjectsResult hasMore nextToken objects prefixes return $ ListObjectsResult hasMore nextToken objects prefixes

View File

@ -136,7 +136,7 @@ putObjectNoSizeTest = funTestWithBucket "PutObject of conduit source with no siz
rFile <- mkRandFile mb70 rFile <- mkRandFile mb70
step "Upload multipart file." step "Upload multipart file."
putObject bucket obj (CB.sourceFile rFile) Nothing putObject bucket obj (CB.sourceFile rFile) Nothing def
step "Retrieve and verify file size" step "Retrieve and verify file size"
destFile <- mkRandFile 0 destFile <- mkRandFile 0
@ -155,7 +155,7 @@ highLevelListingTest = funTestWithBucket "High-level listObjects Test" $
step "put 3 objects" step "put 3 objects"
let expectedObjects = ["dir/o1", "dir/dir1/o2", "dir/dir2/o3"] let expectedObjects = ["dir/o1", "dir/dir1/o2", "dir/dir2/o3"]
forM_ expectedObjects $ forM_ expectedObjects $
\obj -> fPutObject bucket obj "/etc/lsb-release" \obj -> fPutObject bucket obj "/etc/lsb-release" def
step "High-level listing of objects" step "High-level listing of objects"
objects <- listObjects bucket Nothing True $$ sinkList objects <- listObjects bucket Nothing True $$ sinkList
@ -215,7 +215,7 @@ listingTest = funTestWithBucket "Listing Test" $ \step bucket -> do
let objects = (\s ->T.concat ["lsb-release", T.pack (show s)]) <$> [1..10::Int] let objects = (\s ->T.concat ["lsb-release", T.pack (show s)]) <$> [1..10::Int]
forM_ [1..10::Int] $ \s -> forM_ [1..10::Int] $ \s ->
fPutObject bucket (T.concat ["lsb-release", T.pack (show s)]) "/etc/lsb-release" fPutObject bucket (T.concat ["lsb-release", T.pack (show s)]) "/etc/lsb-release" def
step "Simple list" step "Simple list"
res <- listObjects' bucket Nothing Nothing Nothing Nothing res <- listObjects' bucket Nothing Nothing Nothing Nothing
@ -285,7 +285,7 @@ liveServerUnitTests = testGroup "Unit tests against a live server"
let mb80 = 80 * 1024 * 1024 let mb80 = 80 * 1024 * 1024
obj = "mpart" obj = "mpart"
void $ putObjectInternal bucket obj $ ODFile "/dev/zero" (Just mb80) void $ putObjectInternal bucket obj def $ ODFile "/dev/zero" (Just mb80)
step "Retrieve and verify file size" step "Retrieve and verify file size"
destFile <- mkRandFile 0 destFile <- mkRandFile 0
@ -321,6 +321,25 @@ liveServerUnitTests = testGroup "Unit tests against a live server"
uploads <- listIncompleteUploads bucket (Just object) False C.$$ sinkList uploads <- listIncompleteUploads bucket (Just object) False C.$$ sinkList
liftIO $ (null uploads) @? "removeIncompleteUploads didn't complete successfully" liftIO $ (null uploads) @? "removeIncompleteUploads didn't complete successfully"
, funTestWithBucket "putObject contentType tests" $ \step bucket -> do
step "fPutObject content type test"
let object = "xxx-content-type"
size1 = 100 :: Int64
step "create server object with content-type"
inputFile <- mkRandFile size1
fPutObject bucket object inputFile def{
pooContentType = Just "application/javascript"
}
-- retrieve obj info to check
oi <- headObject bucket object
let m = oiMetadata oi
step "Validate content-type"
liftIO $ assertEqual "Content-Type did not match" (Just "application/javascript") (Map.lookup "Content-Type" m)
step "Cleanup actions"
removeObject bucket object
, funTestWithBucket "copyObject related tests" $ \step bucket -> do , funTestWithBucket "copyObject related tests" $ \step bucket -> do
step "copyObjectSingle basic tests" step "copyObjectSingle basic tests"
@ -330,14 +349,17 @@ liveServerUnitTests = testGroup "Unit tests against a live server"
step "create server object to copy" step "create server object to copy"
inputFile <- mkRandFile size1 inputFile <- mkRandFile size1
fPutObject bucket object inputFile fPutObject bucket object inputFile def
step "copy object" step "copy object"
let srcInfo = def { srcBucket = bucket, srcObject = object} let srcInfo = def { srcBucket = bucket, srcObject = object}
(etag, modTime) <- copyObjectSingle bucket objCopy srcInfo [] (etag, modTime) <- copyObjectSingle bucket objCopy srcInfo []
-- retrieve obj info to check -- retrieve obj info to check
ObjectInfo _ t e s <- headObject bucket objCopy oi <- headObject bucket objCopy
let t = oiModTime oi
let e = oiETag oi
let s = oiSize oi
let isMTimeDiffOk = abs (diffUTCTime modTime t) < 1.0 let isMTimeDiffOk = abs (diffUTCTime modTime t) < 1.0
@ -356,7 +378,7 @@ liveServerUnitTests = testGroup "Unit tests against a live server"
let mb15 = 15 * 1024 * 1024 let mb15 = 15 * 1024 * 1024
mb5 = 5 * 1024 * 1024 mb5 = 5 * 1024 * 1024
randFile <- mkRandFile mb15 randFile <- mkRandFile mb15
fPutObject bucket srcObj randFile fPutObject bucket srcObj randFile def
step "create new multipart upload" step "create new multipart upload"
uid <- newMultipartUpload bucket copyObj [] uid <- newMultipartUpload bucket copyObj []
@ -375,7 +397,8 @@ liveServerUnitTests = testGroup "Unit tests against a live server"
void $ completeMultipartUpload bucket copyObj uid parts void $ completeMultipartUpload bucket copyObj uid parts
step "verify copied object size" step "verify copied object size"
(ObjectInfo _ _ _ s') <- headObject bucket copyObj oi <- headObject bucket copyObj
let s' = oiSize oi
liftIO $ (s' == mb15) @? "Size failed to match" liftIO $ (s' == mb15) @? "Size failed to match"
@ -389,8 +412,9 @@ liveServerUnitTests = testGroup "Unit tests against a live server"
sizes = map (* (1024 * 1024)) [15, 65] sizes = map (* (1024 * 1024)) [15, 65]
step "Prepare" step "Prepare"
forM_ (zip srcs sizes) $ \(src, size) -> forM_ (zip srcs sizes) $ \(src, size) -> do
fPutObject bucket src =<< mkRandFile size inputFile <- mkRandFile size
fPutObject bucket src inputFile def
step "make small and large object copy" step "make small and large object copy"
forM_ (zip copyObjs srcs) $ \(cp, src) -> forM_ (zip copyObjs srcs) $ \(cp, src) ->
@ -408,7 +432,8 @@ liveServerUnitTests = testGroup "Unit tests against a live server"
size = 15 * 1024 * 1024 size = 15 * 1024 * 1024
step "Prepare" step "Prepare"
fPutObject bucket src =<< mkRandFile size inputFile <- mkRandFile size
fPutObject bucket src inputFile def
step "copy last 10MiB of object" step "copy last 10MiB of object"
copyObject def { dstBucket = bucket, dstObject = copyObj } def{ copyObject def { dstBucket = bucket, dstObject = copyObj } def{
@ -454,10 +479,10 @@ basicTests = funTestWithBucket "Basic tests" $ \step bucket -> do
liftIO $ region == "us-east-1" @? ("Got unexpected region => " ++ show region) liftIO $ region == "us-east-1" @? ("Got unexpected region => " ++ show region)
step "singlepart putObject works" step "singlepart putObject works"
fPutObject bucket "lsb-release" "/etc/lsb-release" fPutObject bucket "lsb-release" "/etc/lsb-release" def
step "fPutObject onto a non-existent bucket and check for NoSuchBucket exception" step "fPutObject onto a non-existent bucket and check for NoSuchBucket exception"
fpE <- MC.try $ fPutObject "nosuchbucket" "lsb-release" "/etc/lsb-release" fpE <- MC.try $ fPutObject "nosuchbucket" "lsb-release" "/etc/lsb-release" def
case fpE of case fpE of
Left exn -> liftIO $ exn @?= NoSuchBucket Left exn -> liftIO $ exn @?= NoSuchBucket
_ -> return () _ -> return ()
@ -516,7 +541,7 @@ basicTests = funTestWithBucket "Basic tests" $ \step bucket -> do
let object = "sample" let object = "sample"
step "create an object" step "create an object"
inputFile <- mkRandFile 0 inputFile <- mkRandFile 0
fPutObject bucket object inputFile fPutObject bucket object inputFile def
step "get metadata of the object" step "get metadata of the object"
res <- statObject bucket object res <- statObject bucket object

View File

@ -21,6 +21,7 @@ module Network.Minio.XmlParser.Test
import qualified Control.Monad.Catch as MC import qualified Control.Monad.Catch as MC
import Data.Time (fromGregorian) import Data.Time (fromGregorian)
import qualified Data.Map as Map
import Test.Tasty import Test.Tasty
import Test.Tasty.HUnit import Test.Tasty.HUnit
@ -126,7 +127,7 @@ testParseListObjectsResult = do
\</ListBucketResult>" \</ListBucketResult>"
expectedListResult = ListObjectsResult True (Just "opaque") [object1] [] expectedListResult = ListObjectsResult True (Just "opaque") [object1] []
object1 = ObjectInfo "my-image.jpg" modifiedTime1 "\"fba9dede5f27731c9771645a39863328\"" 434234 object1 = ObjectInfo "my-image.jpg" modifiedTime1 "\"fba9dede5f27731c9771645a39863328\"" 434234 Map.empty
modifiedTime1 = flip UTCTime 64230 $ fromGregorian 2009 10 12 modifiedTime1 = flip UTCTime 64230 $ fromGregorian 2009 10 12
parsedListObjectsResult <- tryValidationErr $ parseListObjectsResponse xmldata parsedListObjectsResult <- tryValidationErr $ parseListObjectsResponse xmldata
@ -153,7 +154,7 @@ testParseListObjectsV1Result = do
\</ListBucketResult>" \</ListBucketResult>"
expectedListResult = ListObjectsV1Result True (Just "my-image1.jpg") [object1] [] expectedListResult = ListObjectsV1Result True (Just "my-image1.jpg") [object1] []
object1 = ObjectInfo "my-image.jpg" modifiedTime1 "\"fba9dede5f27731c9771645a39863328\"" 434234 object1 = ObjectInfo "my-image.jpg" modifiedTime1 "\"fba9dede5f27731c9771645a39863328\"" 434234 Map.empty
modifiedTime1 = flip UTCTime 64230 $ fromGregorian 2009 10 12 modifiedTime1 = flip UTCTime 64230 $ fromGregorian 2009 10 12
parsedListObjectsV1Result <- tryValidationErr $ parseListObjectsV1Response xmldata parsedListObjectsV1Result <- tryValidationErr $ parseListObjectsV1Response xmldata