Add getServerInfo admin API (#91)

- Add Admin API helper functions like buildAdminRequest
This commit is contained in:
Krishnan Parthasarathi 2018-05-31 18:06:24 -07:00 committed by Harshavardhana
parent a946dfd305
commit 952c0b0342
8 changed files with 358 additions and 34 deletions

View File

@ -27,9 +27,11 @@ library
hs-source-dirs: src
ghc-options: -Wall
exposed-modules: Network.Minio
, Network.Minio.AdminAPI
, Network.Minio.S3API
other-modules: Lib.Prelude
, Network.Minio.API
, Network.Minio.APICommon
, Network.Minio.Data
, Network.Minio.Data.ByteString
, Network.Minio.Data.Crypto
@ -103,7 +105,9 @@ test-suite minio-hs-live-server-test
, TypeFamilies
other-modules: Lib.Prelude
, Network.Minio
, Network.Minio.AdminAPI
, Network.Minio.API
, Network.Minio.APICommon
, Network.Minio.CopyObject
, Network.Minio.Data
, Network.Minio.Data.ByteString
@ -207,7 +211,9 @@ test-suite minio-hs-test
, TypeFamilies
other-modules: Lib.Prelude
, Network.Minio
, Network.Minio.AdminAPI
, Network.Minio.API
, Network.Minio.APICommon
, Network.Minio.Data
, Network.Minio.Data.ByteString
, Network.Minio.Data.Crypto

View File

@ -1,5 +1,5 @@
--
-- Minio Haskell SDK, (C) 2017 Minio, Inc.
-- Minio Haskell SDK, (C) 2017, 2018 Minio, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
@ -31,7 +31,6 @@ module Network.Minio.API
import qualified Data.ByteString as B
import qualified Data.Char as C
import qualified Data.Conduit as C
import Data.Conduit.Binary (sourceHandleRange)
import Data.Default (def)
import qualified Data.Map as Map
import qualified Data.Text as T
@ -44,32 +43,13 @@ import Network.HTTP.Types.Header (hHost)
import Lib.Prelude
import Network.Minio.APICommon
import Network.Minio.Data
import Network.Minio.Data.Crypto
import Network.Minio.Errors
import Network.Minio.Sign.V4
import Network.Minio.Utils
import Network.Minio.XmlParser
sha256Header :: ByteString -> HT.Header
sha256Header = ("x-amz-content-sha256", )
getPayloadSHA256Hash :: (MonadIO m) => Payload -> m ByteString
getPayloadSHA256Hash (PayloadBS bs) = return $ hashSHA256 bs
getPayloadSHA256Hash (PayloadH h off size) = hashSHA256FromSource $
sourceHandleRange h
(return . fromIntegral $ off)
(return . fromIntegral $ size)
getRequestBody :: Payload -> NC.RequestBody
getRequestBody (PayloadBS bs) = NC.RequestBodyBS bs
getRequestBody (PayloadH h off size) =
NC.requestBodySource (fromIntegral size) $
sourceHandleRange h
(return . fromIntegral $ off)
(return . fromIntegral $ size)
-- | Fetch bucket location (region)
getLocation :: Bucket -> Minio Region
getLocation bucket = do

View File

@ -0,0 +1,44 @@
--
-- Minio Haskell SDK, (C) 2018 Minio, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
module Network.Minio.APICommon where
import Data.Conduit.Binary (sourceHandleRange)
import qualified Network.HTTP.Conduit as NC
import qualified Network.HTTP.Types as HT
import Lib.Prelude
import Network.Minio.Data
import Network.Minio.Data.Crypto
sha256Header :: ByteString -> HT.Header
sha256Header = ("x-amz-content-sha256", )
getPayloadSHA256Hash :: (MonadIO m) => Payload -> m ByteString
getPayloadSHA256Hash (PayloadBS bs) = return $ hashSHA256 bs
getPayloadSHA256Hash (PayloadH h off size) = hashSHA256FromSource $
sourceHandleRange h
(return . fromIntegral $ off)
(return . fromIntegral $ size)
getRequestBody :: Payload -> NC.RequestBody
getRequestBody (PayloadBS bs) = NC.RequestBodyBS bs
getRequestBody (PayloadH h off size) =
NC.requestBodySource (fromIntegral size) $
sourceHandleRange h
(return . fromIntegral $ off)
(return . fromIntegral $ size)

View File

@ -0,0 +1,264 @@
--
-- Minio Haskell SDK, (C) 2018 Minio, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
module Network.Minio.AdminAPI
( ErasureSets(..)
, ErasureInfo(..)
, Backend(..)
, ConnStats(..)
, HttpStats(..)
, ServerProps(..)
, CountNAvgTime(..)
, StorageClass(..)
, StorageInfo(..)
, SIData(..)
, ServerInfo(..)
, getServerInfo
) where
import Data.Aeson (FromJSON, Value (Object),
eitherDecode, parseJSON, withObject,
(.:))
import Data.Aeson.Types (typeMismatch)
import qualified Data.ByteString as B
import qualified Data.Text as T
import Data.Time (NominalDiffTime, getCurrentTime)
import Network.HTTP.Conduit (Response)
import qualified Network.HTTP.Conduit as NC
import qualified Network.HTTP.Types as HT
import Network.HTTP.Types.Header (hHost)
import Lib.Prelude
import Network.Minio.APICommon
import Network.Minio.Data
import Network.Minio.Errors
import Network.Minio.Sign.V4
import Network.Minio.Utils
data ErasureSets = ErasureSets
{ esUuid :: Text
, esEndpoint :: Text
, esState :: Text
} deriving (Eq, Show)
instance FromJSON ErasureSets where
parseJSON = withObject "ErasureSets" $ \v -> ErasureSets
<$> v .: "uuid"
<*> v .: "endpoint"
<*> v .: "state"
data StorageClass = StorageClass
{ scParity :: Int
, scData :: Int
} deriving (Eq, Show)
data ErasureInfo = ErasureInfo
{ eiOnlineDisks :: Int
, eiOfflineDisks :: Int
, eiStandard :: StorageClass
, eiReducedRedundancy :: StorageClass
, eiSets :: [[ErasureSets]]
} deriving (Eq, Show)
instance FromJSON ErasureInfo where
parseJSON = withObject "ErasureInfo" $ \v -> do
onlineDisks <- v .: "OnlineDisks"
offlineDisks <- v .: "OfflineDisks"
stdClass <- StorageClass
<$> v .: "StandardSCData"
<*> v .: "StandardSCParity"
rrClass <- StorageClass
<$> v .: "RRSCData"
<*> v .: "RRSCParity"
sets <- v .: "Sets"
return $ ErasureInfo onlineDisks offlineDisks stdClass rrClass sets
data Backend = BackendFS
| BackendErasure ErasureInfo
deriving (Eq, Show)
instance FromJSON Backend where
parseJSON = withObject "Backend" $ \v -> do
typ <- v .: "Type"
case typ :: Int of
1 -> return BackendFS
2 -> BackendErasure <$> parseJSON (Object v)
_ -> typeMismatch "BackendType" (Object v)
data ConnStats = ConnStats
{ csTransferred :: Int64
, csReceived :: Int64
} deriving (Eq, Show)
instance FromJSON ConnStats where
parseJSON = withObject "ConnStats" $ \v -> ConnStats
<$> v .: "transferred"
<*> v .: "received"
data ServerProps = ServerProps
{ spUptime :: NominalDiffTime
, spVersion :: Text
, spCommitId :: Text
, spRegion :: Text
, spSqsArns :: [Text]
} deriving (Eq, Show)
instance FromJSON ServerProps where
parseJSON = withObject "SIServer" $ \v -> do
uptimeNs <- v .: "uptime"
let uptime = uptimeNs / 1e9
ver <- v .: "version"
commitId <- v .: "commitID"
region <- v .: "region"
arn <- v .: "sqsARN"
return $ ServerProps uptime ver commitId region arn
data StorageInfo = StorageInfo
{ siUsed :: Int64
, siBackend :: Backend
} deriving (Eq, Show)
instance FromJSON StorageInfo where
parseJSON = withObject "StorageInfo" $ \v -> StorageInfo
<$> v .: "Used"
<*> v .: "Backend"
data CountNAvgTime = CountNAvgTime
{ caCount :: Int64
, caAvgDuration :: Text
} deriving (Eq, Show)
instance FromJSON CountNAvgTime where
parseJSON = withObject "CountNAvgTime" $ \v -> CountNAvgTime
<$> v .: "count"
<*> v .: "avgDuration"
data HttpStats = HttpStats
{ hsTotalHeads :: CountNAvgTime
, hsSuccessHeads :: CountNAvgTime
, hsTotalGets :: CountNAvgTime
, hsSuccessGets :: CountNAvgTime
, hsTotalPuts :: CountNAvgTime
, hsSuccessPuts :: CountNAvgTime
, hsTotalPosts :: CountNAvgTime
, hsSuccessPosts :: CountNAvgTime
, hsTotalDeletes :: CountNAvgTime
, hsSuccessDeletes :: CountNAvgTime
} deriving (Eq, Show)
instance FromJSON HttpStats where
parseJSON = withObject "HttpStats" $ \v -> HttpStats
<$> v .: "totalHEADs"
<*> v .: "successHEADs"
<*> v .: "totalGETs"
<*> v .: "successGETs"
<*> v .: "totalPUTs"
<*> v .: "successPUTs"
<*> v .: "totalPOSTs"
<*> v .: "successPOSTs"
<*> v .: "totalDELETEs"
<*> v .: "successDELETEs"
data SIData = SIData
{ sdStorage :: StorageInfo
, sdConnStats :: ConnStats
, sdHttpStats :: HttpStats
, sdProps :: ServerProps
} deriving (Eq, Show)
instance FromJSON SIData where
parseJSON = withObject "SIData" $ \v -> SIData
<$> v .: "storage"
<*> v .: "network"
<*> v .: "http"
<*> v .: "server"
data ServerInfo = ServerInfo
{ siError :: Text
, siAddr :: Text
, siData :: SIData
} deriving (Eq, Show)
instance FromJSON ServerInfo where
parseJSON = withObject "ServerInfo" $ \v -> do
err <- v .: "error"
addr <- v .: "addr"
d <- v .: "data"
return $ ServerInfo err addr d
adminPath :: ByteString
adminPath = "/minio/admin"
getServerInfo :: Minio [ServerInfo]
getServerInfo = do
rsp <- executeAdminRequest AdminReqInfo { ariMethod = HT.methodGet
, ariPayload = PayloadBS B.empty
, ariPayloadHash = Nothing
, ariPath = "v1/info"
, ariHeaders = []
, ariQueryParams = []
}
let rspBS = NC.responseBody rsp
case eitherDecode rspBS of
Right si -> return si
Left err -> throwIO $ MErrVJsonParse $ T.pack err
executeAdminRequest :: AdminReqInfo -> Minio (Response LByteString)
executeAdminRequest ari = do
req <- buildAdminRequest ari
mgr <- asks mcConnManager
httpLbs req mgr
buildAdminRequest :: AdminReqInfo -> Minio NC.Request
buildAdminRequest areq = do
ci <- asks mcConnInfo
sha256Hash <- if | connectIsSecure ci ->
-- if secure connection
return "UNSIGNED-PAYLOAD"
-- otherwise compute sha256
| otherwise -> getPayloadSHA256Hash (ariPayload areq)
timeStamp <- liftIO getCurrentTime
let hostHeader = (hHost, getHostAddr ci)
newAreq = areq { ariPayloadHash = Just sha256Hash
, ariHeaders = hostHeader
: sha256Header sha256Hash
: ariHeaders areq
}
signReq = toRequest ci newAreq
sp = SignParams (connectAccessKey ci) (connectSecretKey ci)
timeStamp Nothing Nothing (ariPayloadHash newAreq)
signHeaders = signV4 sp signReq
-- Update signReq with Authorization header containing v4 signature
return signReq {
NC.requestHeaders = ariHeaders newAreq ++ mkHeaderFromPairs signHeaders
}
where
toRequest :: ConnectInfo -> AdminReqInfo -> NC.Request
toRequest ci aReq = NC.defaultRequest
{ NC.method = ariMethod aReq
, NC.secure = connectIsSecure ci
, NC.host = encodeUtf8 $ connectHost ci
, NC.port = connectPort ci
, NC.path = B.intercalate "/" [adminPath, ariPath aReq]
, NC.requestHeaders = ariHeaders aReq
, NC.queryString = HT.renderQuery False $ ariQueryParams aReq
, NC.requestBody = getRequestBody (ariPayload aReq)
}

View File

@ -1,5 +1,5 @@
--
-- Minio Haskell SDK, (C) 2017 Minio, Inc.
-- Minio Haskell SDK, (C) 2017, 2018 Minio, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
@ -472,6 +472,15 @@ data Payload = PayloadBS ByteString
instance Default Payload where
def = PayloadBS ""
data AdminReqInfo = AdminReqInfo {
ariMethod :: Method
, ariPayloadHash :: Maybe ByteString
, ariPayload :: Payload
, ariPath :: ByteString
, ariHeaders :: [Header]
, ariQueryParams :: Query
}
data S3ReqInfo = S3ReqInfo {
riMethod :: Method
, riBucket :: Maybe Bucket

View File

@ -1,5 +1,5 @@
--
-- Minio Haskell SDK, (C) 2017 Minio, Inc.
-- Minio Haskell SDK, (C) 2017, 2018 Minio, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
@ -38,6 +38,7 @@ data MErrV = MErrVSinglePUTSizeExceeded Int64
| MErrVInvalidBucketName Text
| MErrVInvalidObjectName Text
| MErrVInvalidUrlExpiry Int
| MErrVJsonParse Text
deriving (Show, Eq)
instance Exception MErrV

View File

@ -1,5 +1,5 @@
--
-- Minio Haskell SDK, (C) 2017 Minio, Inc.
-- Minio Haskell SDK, (C) 2017, 2018 Minio, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
@ -17,13 +17,16 @@
module Network.Minio.API.Test
( bucketNameValidityTests
, objectNameValidityTests
, parseJSONTests
) where
import Data.Aeson (eitherDecode)
import Test.Tasty
import Test.Tasty.HUnit
import Lib.Prelude
import Network.Minio.AdminAPI
import Network.Minio.API
assertBool' :: Bool -> Assertion
@ -49,3 +52,18 @@ objectNameValidityTests = testGroup "Object Name Validity Tests"
[ testCase "Empty name" $ assertBool' $ not $ isValidObjectName ""
, testCase "Has unicode characters" $ assertBool' $ isValidObjectName "日本国"
]
parseJSONTests :: TestTree
parseJSONTests = testGroup "Parse Minio Admin API JSON parser Tests" $
map (\(tName, tDesc, tfn, tVal) -> testCase tName $ assertBool tDesc $
tfn (eitherDecode tVal :: Either [Char] [ServerInfo])) testCases
where
testCases = [ ("FSBackend", "Verify server info json parsing for FS backend", isRight, fsJSON)
, ("Erasure Backend", "Verify server info json parsing for Erasure backend", isRight, erasureJSON)
, ("Unknown Backend", "Verify server info json parsing for invalid backend", isLeft, invalidJSON)
]
fsJSON = "[{\"error\":\"\",\"addr\":\"192.168.1.218:9000\",\"data\":{\"storage\":{\"Used\":20530,\"Backend\":{\"Type\":1,\"OnlineDisks\":0,\"OfflineDisks\":0,\"StandardSCData\":0,\"StandardSCParity\":0,\"RRSCData\":0,\"RRSCParity\":0,\"Sets\":null}},\"network\":{\"transferred\":808,\"received\":1160},\"http\":{\"totalHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalGETs\":{\"count\":1,\"avgDuration\":\"0s\"},\"successGETs\":{\"count\":1,\"avgDuration\":\"0s\"},\"totalPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"}},\"server\":{\"uptime\":5992503019270,\"version\":\"DEVELOPMENT.GOGET\",\"commitID\":\"DEVELOPMENT.GOGET\",\"region\":\"\",\"sqsARN\":[]}}}]"
erasureJSON = "[{\"error\":\"\",\"addr\":\"192.168.1.218:9000\",\"data\":{\"storage\":{\"Used\":83084,\"Backend\":{\"Type\":2,\"OnlineDisks\":4,\"OfflineDisks\":0,\"StandardSCData\":2,\"StandardSCParity\":2,\"RRSCData\":2,\"RRSCParity\":2,\"Sets\":[[{\"uuid\":\"16ec6f2c-9197-4787-904a-36bb2c2683f8\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"4052e086-ef99-4aa5-ae2b-8e27559432f6\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"d0639950-ddd3-45b0-93ca-fd86f5d79f72\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"30ec68c0-37e1-4592-82c1-26b143c0ac10\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]]}},\"network\":{\"transferred\":404,\"received\":0},\"http\":{\"totalHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalGETs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successGETs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"}},\"server\":{\"uptime\":2738903073,\"version\":\"DEVELOPMENT.GOGET\",\"commitID\":\"DEVELOPMENT.GOGET\",\"region\":\"\",\"sqsARN\":[]}}}]"
invalidJSON = "[{\"error\":\"\",\"addr\":\"192.168.1.218:9000\",\"data\":{\"storage\":{\"Used\":83084,\"Backend\":{\"Type\":42,\"OnlineDisks\":4,\"OfflineDisks\":0,\"StandardSCData\":2,\"StandardSCParity\":2,\"RRSCData\":2,\"RRSCParity\":2,\"Sets\":[[{\"uuid\":\"16ec6f2c-9197-4787-904a-36bb2c2683f8\",\"endpoint\":\"/tmp/1\",\"state\":\"ok\"},{\"uuid\":\"4052e086-ef99-4aa5-ae2b-8e27559432f6\",\"endpoint\":\"/tmp/2\",\"state\":\"ok\"},{\"uuid\":\"d0639950-ddd3-45b0-93ca-fd86f5d79f72\",\"endpoint\":\"/tmp/3\",\"state\":\"ok\"},{\"uuid\":\"30ec68c0-37e1-4592-82c1-26b143c0ac10\",\"endpoint\":\"/tmp/4\",\"state\":\"ok\"}]]}},\"network\":{\"transferred\":404,\"received\":0},\"http\":{\"totalHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successHEADs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalGETs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successGETs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPUTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successPOSTs\":{\"count\":0,\"avgDuration\":\"0s\"},\"totalDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"},\"successDELETEs\":{\"count\":0,\"avgDuration\":\"0s\"}},\"server\":{\"uptime\":2738903073,\"version\":\"DEVELOPMENT.GOGET\",\"commitID\":\"DEVELOPMENT.GOGET\",\"region\":\"\",\"sqsARN\":[]}}}]"

View File

@ -1,5 +1,5 @@
--
-- Minio Haskell SDK, (C) 2017 Minio, Inc.
-- Minio Haskell SDK, (C) 2017, 2018 Minio, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
@ -15,9 +15,9 @@
--
import Test.Tasty
import Test.Tasty.QuickCheck as QC
import Test.Tasty.QuickCheck as QC
import qualified Data.List as L
import qualified Data.List as L
import Lib.Prelude
@ -57,8 +57,8 @@ qcProps = testGroup "(checked by QuickCheck)"
-- check that pns increments from 1.
isPNumsAscendingFrom1 = all (\(a, b) -> a == b) $ zip pns [1..]
consPairs [] = []
consPairs [_] = []
consPairs [] = []
consPairs [_] = []
consPairs (a:(b:c)) = (a, b):(consPairs (b:c))
-- check `offs` is monotonically increasing.
@ -114,7 +114,9 @@ qcProps = testGroup "(checked by QuickCheck)"
]
unitTests :: TestTree
unitTests = testGroup "Unit tests" [xmlGeneratorTests, xmlParserTests,
bucketNameValidityTests,
objectNameValidityTests,
limitedMapConcurrentlyTests]
unitTests = testGroup "Unit tests" [ xmlGeneratorTests, xmlParserTests
, bucketNameValidityTests
, objectNameValidityTests
, parseJSONTests
, limitedMapConcurrentlyTests
]