Remove system-file(path/io)

This commit is contained in:
Michael Snoyman 2018-06-21 19:30:48 +03:00
parent 014114855b
commit 96e9a53a17
No known key found for this signature in database
GPG Key ID: A048E8C057E86876
4 changed files with 34 additions and 39 deletions

View File

@ -39,8 +39,6 @@ dependencies:
- persistent-template - persistent-template
- resourcet - resourcet
- shakespeare - shakespeare
- system-fileio
- system-filepath
- tar - tar
- template-haskell - template-haskell
- temporary - temporary

View File

@ -7,7 +7,7 @@ import ClassyPrelude.Yesod
import Control.Monad.State.Strict (modify, execStateT) import Control.Monad.State.Strict (modify, execStateT)
import qualified Data.HashMap.Strict as HashMap import qualified Data.HashMap.Strict as HashMap
import qualified Data.Yaml as Yaml import qualified Data.Yaml as Yaml
import Filesystem (readTextFile, isFile) import System.Directory
import Types import Types
@ -37,7 +37,7 @@ readGhcLinks dir = do
path = dir path = dir
</> unpack (toPathPiece arch) </> unpack (toPathPiece arch)
</> unpack fileName </> unpack fileName
whenM (liftIO $ isFile (fromString path)) $ do whenM (liftIO $ doesFileExist path) $ do
text <- liftIO $ readTextFile (fromString path) text <- liftIO $ readFileUtf8 path
modify (HashMap.insert (arch, ver) text) modify (HashMap.insert (arch, ver) text)
return $ GhcLinks hashMap return $ GhcLinks hashMap

View File

@ -53,13 +53,12 @@ import CMarkGFM
import System.Directory (removeFile) import System.Directory (removeFile)
import Stackage.Database.Haddock import Stackage.Database.Haddock
import System.FilePath (takeBaseName, takeExtension) import System.FilePath (takeBaseName, takeExtension)
import ClassyPrelude.Conduit hiding (pi, FilePath, (</>)) import ClassyPrelude.Conduit hiding (pi)
import Text.Blaze.Html (Html, toHtml, preEscapedToHtml) import Text.Blaze.Html (Html, toHtml, preEscapedToHtml)
import Yesod.Form.Fields (Textarea (..)) import Yesod.Form.Fields (Textarea (..))
import Stackage.Database.Types import Stackage.Database.Types
import System.Directory (getAppUserDataDirectory) import System.Directory (getAppUserDataDirectory, doesDirectoryExist, createDirectoryIfMissing)
import qualified Filesystem as F import System.FilePath (takeFileName, takeDirectory)
import Filesystem.Path.CurrentOS (filename, directory, FilePath, encodeString, (</>))
import Data.Conduit.Process import Data.Conduit.Process
import Stackage.Types import Stackage.Types
import Stackage.Metadata import Stackage.Metadata
@ -182,23 +181,23 @@ sourceBuildPlans :: MonadResource m => FilePath -> ConduitT i (SnapName, FilePat
sourceBuildPlans root = do sourceBuildPlans root = do
forM_ ["lts-haskell", "stackage-nightly"] $ \repoName -> do forM_ ["lts-haskell", "stackage-nightly"] $ \repoName -> do
dir <- liftIO $ cloneOrUpdate root "fpco" repoName dir <- liftIO $ cloneOrUpdate root "fpco" repoName
sourceDirectory (encodeString dir) .| concatMapMC (go Left . fromString) sourceDirectory dir .| concatMapMC (go Left . fromString)
let docdir = dir </> "docs" let docdir = dir </> "docs"
whenM (liftIO $ F.isDirectory docdir) $ whenM (liftIO $ doesDirectoryExist docdir) $
sourceDirectory (encodeString docdir) .| concatMapMC (go Right . fromString) sourceDirectory docdir .| concatMapMC (go Right . fromString)
where where
go wrapper fp | Just name <- nameFromFP fp = liftIO $ do go wrapper fp | Just name <- nameFromFP fp = liftIO $ do
let bp = decodeFileEither (encodeString fp) >>= either throwIO return let bp = decodeFileEither fp >>= either throwIO return
return $ Just (name, fp, wrapper bp) return $ Just (name, fp, wrapper bp)
go _ _ = return Nothing go _ _ = return Nothing
nameFromFP fp = do nameFromFP fp = do
base <- stripSuffix ".yaml" $ pack $ encodeString $ filename fp base <- stripSuffix ".yaml" $ pack $ takeFileName fp
fromPathPiece base fromPathPiece base
cloneOrUpdate :: FilePath -> String -> String -> IO FilePath cloneOrUpdate :: FilePath -> String -> String -> IO FilePath
cloneOrUpdate root org name = do cloneOrUpdate root org name = do
exists <- F.isDirectory dest exists <- doesDirectoryExist dest
if exists if exists
then do then do
let git = runIn dest "git" let git = runIn dest "git"
@ -214,7 +213,7 @@ runIn :: FilePath -> String -> [String] -> IO ()
runIn dir cmd args = runIn dir cmd args =
withCheckedProcess cp $ \ClosedStream Inherited Inherited -> return () withCheckedProcess cp $ \ClosedStream Inherited Inherited -> return ()
where where
cp = (proc cmd args) { cwd = Just $ encodeString dir } cp = (proc cmd args) { cwd = Just dir }
openStackageDatabase :: MonadIO m => PostgresConf -> m StackageDatabase openStackageDatabase :: MonadIO m => PostgresConf -> m StackageDatabase
openStackageDatabase pg = liftIO $ do openStackageDatabase pg = liftIO $ do
@ -244,8 +243,8 @@ createStackageDatabase fp = liftIO $ do
runMigration migrateAll runMigration migrateAll
unless schemaMatch $ insert_ $ Schema currentSchema unless schemaMatch $ insert_ $ Schema currentSchema
root <- liftIO $ fmap (</> fromString "database") $ fmap fromString $ getAppUserDataDirectory "stackage" root <- liftIO $ (</> "database") <$> getAppUserDataDirectory "stackage"
F.createTree root createDirectoryIfMissing True root
runResourceT $ do runResourceT $ do
putStrLn "Updating all-cabal-metadata repo" putStrLn "Updating all-cabal-metadata repo"
flip runSqlPool pool $ runConduit $ sourcePackages root .| getZipSink flip runSqlPool pool $ runConduit $ sourcePackages root .| getZipSink
@ -369,9 +368,9 @@ addPlan name fp bp = do
[ "log" [ "log"
, "--format=%ad" , "--format=%ad"
, "--date=short" , "--date=short"
, encodeString $ filename fp , takeFileName fp
] ]
cp = cp' { cwd = Just $ encodeString $ directory fp } cp = cp' { cwd = Just $ takeDirectory fp }
t <- withCheckedProcess cp $ \ClosedStream out ClosedStream -> t <- withCheckedProcess cp $ \ClosedStream out ClosedStream ->
runConduit $ out .| decodeUtf8C .| foldC runConduit $ out .| decodeUtf8C .| foldC
case readMay $ concat $ take 1 $ words t of case readMay $ concat $ take 1 $ words t of

View File

@ -11,9 +11,8 @@ import qualified Codec.Archive.Tar as Tar
import Stackage.Database import Stackage.Database
import Network.HTTP.Client import Network.HTTP.Client
import Network.HTTP.Client.Conduit (bodyReaderSource) import Network.HTTP.Client.Conduit (bodyReaderSource)
import Filesystem (rename, removeTree, removeFile, isFile, createTree) import System.Directory
import Web.PathPieces (toPathPiece) import Web.PathPieces (toPathPiece)
import Filesystem.Path.CurrentOS (parent, fromText, encodeString)
import Network.HTTP.Types (status200) import Network.HTTP.Types (status200)
import Data.Streaming.Network (bindPortTCP) import Data.Streaming.Network (bindPortTCP)
import Network.AWS (Credentials (Discover), newEnv, import Network.AWS (Credentials (Discover), newEnv,
@ -30,10 +29,9 @@ import qualified Data.Conduit.Binary as CB
import Data.Conduit.Zlib (WindowBits (WindowBits), import Data.Conduit.Zlib (WindowBits (WindowBits),
compress, ungzip) compress, ungzip)
import qualified Hoogle import qualified Hoogle
import System.Directory (getAppUserDataDirectory)
import Control.SingleRun import Control.SingleRun
import qualified Data.ByteString.Lazy as L import qualified Data.ByteString.Lazy as L
import System.FilePath (splitPath) import System.FilePath (splitPath, takeDirectory)
import System.Environment (getEnv) import System.Environment (getEnv)
hoogleKey :: SnapName -> Text hoogleKey :: SnapName -> Text
@ -55,24 +53,24 @@ newHoogleLocker :: Bool -- ^ print exceptions?
-> Manager -> Manager
-> IO (SingleRun SnapName (Maybe FilePath)) -> IO (SingleRun SnapName (Maybe FilePath))
newHoogleLocker toPrint man = mkSingleRun $ \name -> do newHoogleLocker toPrint man = mkSingleRun $ \name -> do
let fp = fromText $ hoogleKey name let fp = unpack $ hoogleKey name
fptmp = encodeString fp <.> "tmp" fptmp = fp <.> "tmp"
exists <- isFile fp exists <- doesFileExist fp
if exists if exists
then return $ Just (encodeString fp) then return $ Just fp
else do else do
req' <- parseRequest $ unpack $ hoogleUrl name req' <- parseRequest $ unpack $ hoogleUrl name
let req = req' { decompress = const False } let req = req' { decompress = const False }
withResponse req man $ \res -> if responseStatus res == status200 withResponse req man $ \res -> if responseStatus res == status200
then do then do
createTree $ parent (fromString fptmp) createDirectoryIfMissing True $ takeDirectory fptmp
runConduitRes runConduitRes
$ bodyReaderSource (responseBody res) $ bodyReaderSource (responseBody res)
.| ungzip .| ungzip
.| sinkFile fptmp .| sinkFile fptmp
rename (fromString fptmp) fp renamePath fptmp fp
return $ Just $ encodeString fp return $ Just fp
else do else do
when toPrint $ mapM brRead res >>= print when toPrint $ mapM brRead res >>= print
return Nothing return Nothing
@ -139,8 +137,8 @@ stackageServerCron = do
let key = hoogleKey name let key = hoogleKey name
upload fp (ObjectKey key) upload fp (ObjectKey key)
let dest = unpack key let dest = unpack key
createTree $ parent (fromString dest) createDirectoryIfMissing True $ takeDirectory dest
rename (fromString fp) (fromString dest) renamePath fp dest
createHoogleDB :: StackageDatabase -> Manager -> SnapName -> IO (Maybe FilePath) createHoogleDB :: StackageDatabase -> Manager -> SnapName -> IO (Maybe FilePath)
createHoogleDB db man name = handleAny (\e -> print e $> Nothing) $ do createHoogleDB db man name = handleAny (\e -> print e $> Nothing) $ do
@ -148,17 +146,17 @@ createHoogleDB db man name = handleAny (\e -> print e $> Nothing) $ do
req' <- parseRequest $ unpack tarUrl req' <- parseRequest $ unpack tarUrl
let req = req' { decompress = const True } let req = req' { decompress = const True }
unlessM (isFile (fromString tarFP)) $ withResponse req man $ \res -> do unlessM (doesFileExist tarFP) $ withResponse req man $ \res -> do
let tmp = tarFP <.> "tmp" let tmp = tarFP <.> "tmp"
createTree $ parent (fromString tmp) createDirectoryIfMissing True $ takeDirectory tmp
runConduitRes runConduitRes
$ bodyReaderSource (responseBody res) $ bodyReaderSource (responseBody res)
.| sinkFile tmp .| sinkFile tmp
rename (fromString tmp) (fromString tarFP) renamePath tmp tarFP
void $ tryIO $ removeTree (fromString bindir) void $ tryIO $ removeDirectoryRecursive bindir
void $ tryIO $ removeFile (fromString outname) void $ tryIO $ removeFile outname
createTree (fromString bindir) createDirectoryIfMissing True bindir
withSystemTempDirectory ("hoogle-" ++ unpack (toPathPiece name)) $ \tmpdir -> do withSystemTempDirectory ("hoogle-" ++ unpack (toPathPiece name)) $ \tmpdir -> do
allPackagePairs <- runConduitRes allPackagePairs <- runConduitRes