mirror of
https://github.com/commercialhaskell/stackage-server.git
synced 2026-01-29 20:30:25 +01:00
Remove system-file(path/io)
This commit is contained in:
parent
014114855b
commit
96e9a53a17
@ -39,8 +39,6 @@ dependencies:
|
|||||||
- persistent-template
|
- persistent-template
|
||||||
- resourcet
|
- resourcet
|
||||||
- shakespeare
|
- shakespeare
|
||||||
- system-fileio
|
|
||||||
- system-filepath
|
|
||||||
- tar
|
- tar
|
||||||
- template-haskell
|
- template-haskell
|
||||||
- temporary
|
- temporary
|
||||||
|
|||||||
@ -7,7 +7,7 @@ import ClassyPrelude.Yesod
|
|||||||
import Control.Monad.State.Strict (modify, execStateT)
|
import Control.Monad.State.Strict (modify, execStateT)
|
||||||
import qualified Data.HashMap.Strict as HashMap
|
import qualified Data.HashMap.Strict as HashMap
|
||||||
import qualified Data.Yaml as Yaml
|
import qualified Data.Yaml as Yaml
|
||||||
import Filesystem (readTextFile, isFile)
|
import System.Directory
|
||||||
|
|
||||||
import Types
|
import Types
|
||||||
|
|
||||||
@ -37,7 +37,7 @@ readGhcLinks dir = do
|
|||||||
path = dir
|
path = dir
|
||||||
</> unpack (toPathPiece arch)
|
</> unpack (toPathPiece arch)
|
||||||
</> unpack fileName
|
</> unpack fileName
|
||||||
whenM (liftIO $ isFile (fromString path)) $ do
|
whenM (liftIO $ doesFileExist path) $ do
|
||||||
text <- liftIO $ readTextFile (fromString path)
|
text <- liftIO $ readFileUtf8 path
|
||||||
modify (HashMap.insert (arch, ver) text)
|
modify (HashMap.insert (arch, ver) text)
|
||||||
return $ GhcLinks hashMap
|
return $ GhcLinks hashMap
|
||||||
|
|||||||
@ -53,13 +53,12 @@ import CMarkGFM
|
|||||||
import System.Directory (removeFile)
|
import System.Directory (removeFile)
|
||||||
import Stackage.Database.Haddock
|
import Stackage.Database.Haddock
|
||||||
import System.FilePath (takeBaseName, takeExtension)
|
import System.FilePath (takeBaseName, takeExtension)
|
||||||
import ClassyPrelude.Conduit hiding (pi, FilePath, (</>))
|
import ClassyPrelude.Conduit hiding (pi)
|
||||||
import Text.Blaze.Html (Html, toHtml, preEscapedToHtml)
|
import Text.Blaze.Html (Html, toHtml, preEscapedToHtml)
|
||||||
import Yesod.Form.Fields (Textarea (..))
|
import Yesod.Form.Fields (Textarea (..))
|
||||||
import Stackage.Database.Types
|
import Stackage.Database.Types
|
||||||
import System.Directory (getAppUserDataDirectory)
|
import System.Directory (getAppUserDataDirectory, doesDirectoryExist, createDirectoryIfMissing)
|
||||||
import qualified Filesystem as F
|
import System.FilePath (takeFileName, takeDirectory)
|
||||||
import Filesystem.Path.CurrentOS (filename, directory, FilePath, encodeString, (</>))
|
|
||||||
import Data.Conduit.Process
|
import Data.Conduit.Process
|
||||||
import Stackage.Types
|
import Stackage.Types
|
||||||
import Stackage.Metadata
|
import Stackage.Metadata
|
||||||
@ -182,23 +181,23 @@ sourceBuildPlans :: MonadResource m => FilePath -> ConduitT i (SnapName, FilePat
|
|||||||
sourceBuildPlans root = do
|
sourceBuildPlans root = do
|
||||||
forM_ ["lts-haskell", "stackage-nightly"] $ \repoName -> do
|
forM_ ["lts-haskell", "stackage-nightly"] $ \repoName -> do
|
||||||
dir <- liftIO $ cloneOrUpdate root "fpco" repoName
|
dir <- liftIO $ cloneOrUpdate root "fpco" repoName
|
||||||
sourceDirectory (encodeString dir) .| concatMapMC (go Left . fromString)
|
sourceDirectory dir .| concatMapMC (go Left . fromString)
|
||||||
let docdir = dir </> "docs"
|
let docdir = dir </> "docs"
|
||||||
whenM (liftIO $ F.isDirectory docdir) $
|
whenM (liftIO $ doesDirectoryExist docdir) $
|
||||||
sourceDirectory (encodeString docdir) .| concatMapMC (go Right . fromString)
|
sourceDirectory docdir .| concatMapMC (go Right . fromString)
|
||||||
where
|
where
|
||||||
go wrapper fp | Just name <- nameFromFP fp = liftIO $ do
|
go wrapper fp | Just name <- nameFromFP fp = liftIO $ do
|
||||||
let bp = decodeFileEither (encodeString fp) >>= either throwIO return
|
let bp = decodeFileEither fp >>= either throwIO return
|
||||||
return $ Just (name, fp, wrapper bp)
|
return $ Just (name, fp, wrapper bp)
|
||||||
go _ _ = return Nothing
|
go _ _ = return Nothing
|
||||||
|
|
||||||
nameFromFP fp = do
|
nameFromFP fp = do
|
||||||
base <- stripSuffix ".yaml" $ pack $ encodeString $ filename fp
|
base <- stripSuffix ".yaml" $ pack $ takeFileName fp
|
||||||
fromPathPiece base
|
fromPathPiece base
|
||||||
|
|
||||||
cloneOrUpdate :: FilePath -> String -> String -> IO FilePath
|
cloneOrUpdate :: FilePath -> String -> String -> IO FilePath
|
||||||
cloneOrUpdate root org name = do
|
cloneOrUpdate root org name = do
|
||||||
exists <- F.isDirectory dest
|
exists <- doesDirectoryExist dest
|
||||||
if exists
|
if exists
|
||||||
then do
|
then do
|
||||||
let git = runIn dest "git"
|
let git = runIn dest "git"
|
||||||
@ -214,7 +213,7 @@ runIn :: FilePath -> String -> [String] -> IO ()
|
|||||||
runIn dir cmd args =
|
runIn dir cmd args =
|
||||||
withCheckedProcess cp $ \ClosedStream Inherited Inherited -> return ()
|
withCheckedProcess cp $ \ClosedStream Inherited Inherited -> return ()
|
||||||
where
|
where
|
||||||
cp = (proc cmd args) { cwd = Just $ encodeString dir }
|
cp = (proc cmd args) { cwd = Just dir }
|
||||||
|
|
||||||
openStackageDatabase :: MonadIO m => PostgresConf -> m StackageDatabase
|
openStackageDatabase :: MonadIO m => PostgresConf -> m StackageDatabase
|
||||||
openStackageDatabase pg = liftIO $ do
|
openStackageDatabase pg = liftIO $ do
|
||||||
@ -244,8 +243,8 @@ createStackageDatabase fp = liftIO $ do
|
|||||||
runMigration migrateAll
|
runMigration migrateAll
|
||||||
unless schemaMatch $ insert_ $ Schema currentSchema
|
unless schemaMatch $ insert_ $ Schema currentSchema
|
||||||
|
|
||||||
root <- liftIO $ fmap (</> fromString "database") $ fmap fromString $ getAppUserDataDirectory "stackage"
|
root <- liftIO $ (</> "database") <$> getAppUserDataDirectory "stackage"
|
||||||
F.createTree root
|
createDirectoryIfMissing True root
|
||||||
runResourceT $ do
|
runResourceT $ do
|
||||||
putStrLn "Updating all-cabal-metadata repo"
|
putStrLn "Updating all-cabal-metadata repo"
|
||||||
flip runSqlPool pool $ runConduit $ sourcePackages root .| getZipSink
|
flip runSqlPool pool $ runConduit $ sourcePackages root .| getZipSink
|
||||||
@ -369,9 +368,9 @@ addPlan name fp bp = do
|
|||||||
[ "log"
|
[ "log"
|
||||||
, "--format=%ad"
|
, "--format=%ad"
|
||||||
, "--date=short"
|
, "--date=short"
|
||||||
, encodeString $ filename fp
|
, takeFileName fp
|
||||||
]
|
]
|
||||||
cp = cp' { cwd = Just $ encodeString $ directory fp }
|
cp = cp' { cwd = Just $ takeDirectory fp }
|
||||||
t <- withCheckedProcess cp $ \ClosedStream out ClosedStream ->
|
t <- withCheckedProcess cp $ \ClosedStream out ClosedStream ->
|
||||||
runConduit $ out .| decodeUtf8C .| foldC
|
runConduit $ out .| decodeUtf8C .| foldC
|
||||||
case readMay $ concat $ take 1 $ words t of
|
case readMay $ concat $ take 1 $ words t of
|
||||||
|
|||||||
@ -11,9 +11,8 @@ import qualified Codec.Archive.Tar as Tar
|
|||||||
import Stackage.Database
|
import Stackage.Database
|
||||||
import Network.HTTP.Client
|
import Network.HTTP.Client
|
||||||
import Network.HTTP.Client.Conduit (bodyReaderSource)
|
import Network.HTTP.Client.Conduit (bodyReaderSource)
|
||||||
import Filesystem (rename, removeTree, removeFile, isFile, createTree)
|
import System.Directory
|
||||||
import Web.PathPieces (toPathPiece)
|
import Web.PathPieces (toPathPiece)
|
||||||
import Filesystem.Path.CurrentOS (parent, fromText, encodeString)
|
|
||||||
import Network.HTTP.Types (status200)
|
import Network.HTTP.Types (status200)
|
||||||
import Data.Streaming.Network (bindPortTCP)
|
import Data.Streaming.Network (bindPortTCP)
|
||||||
import Network.AWS (Credentials (Discover), newEnv,
|
import Network.AWS (Credentials (Discover), newEnv,
|
||||||
@ -30,10 +29,9 @@ import qualified Data.Conduit.Binary as CB
|
|||||||
import Data.Conduit.Zlib (WindowBits (WindowBits),
|
import Data.Conduit.Zlib (WindowBits (WindowBits),
|
||||||
compress, ungzip)
|
compress, ungzip)
|
||||||
import qualified Hoogle
|
import qualified Hoogle
|
||||||
import System.Directory (getAppUserDataDirectory)
|
|
||||||
import Control.SingleRun
|
import Control.SingleRun
|
||||||
import qualified Data.ByteString.Lazy as L
|
import qualified Data.ByteString.Lazy as L
|
||||||
import System.FilePath (splitPath)
|
import System.FilePath (splitPath, takeDirectory)
|
||||||
import System.Environment (getEnv)
|
import System.Environment (getEnv)
|
||||||
|
|
||||||
hoogleKey :: SnapName -> Text
|
hoogleKey :: SnapName -> Text
|
||||||
@ -55,24 +53,24 @@ newHoogleLocker :: Bool -- ^ print exceptions?
|
|||||||
-> Manager
|
-> Manager
|
||||||
-> IO (SingleRun SnapName (Maybe FilePath))
|
-> IO (SingleRun SnapName (Maybe FilePath))
|
||||||
newHoogleLocker toPrint man = mkSingleRun $ \name -> do
|
newHoogleLocker toPrint man = mkSingleRun $ \name -> do
|
||||||
let fp = fromText $ hoogleKey name
|
let fp = unpack $ hoogleKey name
|
||||||
fptmp = encodeString fp <.> "tmp"
|
fptmp = fp <.> "tmp"
|
||||||
|
|
||||||
exists <- isFile fp
|
exists <- doesFileExist fp
|
||||||
if exists
|
if exists
|
||||||
then return $ Just (encodeString fp)
|
then return $ Just fp
|
||||||
else do
|
else do
|
||||||
req' <- parseRequest $ unpack $ hoogleUrl name
|
req' <- parseRequest $ unpack $ hoogleUrl name
|
||||||
let req = req' { decompress = const False }
|
let req = req' { decompress = const False }
|
||||||
withResponse req man $ \res -> if responseStatus res == status200
|
withResponse req man $ \res -> if responseStatus res == status200
|
||||||
then do
|
then do
|
||||||
createTree $ parent (fromString fptmp)
|
createDirectoryIfMissing True $ takeDirectory fptmp
|
||||||
runConduitRes
|
runConduitRes
|
||||||
$ bodyReaderSource (responseBody res)
|
$ bodyReaderSource (responseBody res)
|
||||||
.| ungzip
|
.| ungzip
|
||||||
.| sinkFile fptmp
|
.| sinkFile fptmp
|
||||||
rename (fromString fptmp) fp
|
renamePath fptmp fp
|
||||||
return $ Just $ encodeString fp
|
return $ Just fp
|
||||||
else do
|
else do
|
||||||
when toPrint $ mapM brRead res >>= print
|
when toPrint $ mapM brRead res >>= print
|
||||||
return Nothing
|
return Nothing
|
||||||
@ -139,8 +137,8 @@ stackageServerCron = do
|
|||||||
let key = hoogleKey name
|
let key = hoogleKey name
|
||||||
upload fp (ObjectKey key)
|
upload fp (ObjectKey key)
|
||||||
let dest = unpack key
|
let dest = unpack key
|
||||||
createTree $ parent (fromString dest)
|
createDirectoryIfMissing True $ takeDirectory dest
|
||||||
rename (fromString fp) (fromString dest)
|
renamePath fp dest
|
||||||
|
|
||||||
createHoogleDB :: StackageDatabase -> Manager -> SnapName -> IO (Maybe FilePath)
|
createHoogleDB :: StackageDatabase -> Manager -> SnapName -> IO (Maybe FilePath)
|
||||||
createHoogleDB db man name = handleAny (\e -> print e $> Nothing) $ do
|
createHoogleDB db man name = handleAny (\e -> print e $> Nothing) $ do
|
||||||
@ -148,17 +146,17 @@ createHoogleDB db man name = handleAny (\e -> print e $> Nothing) $ do
|
|||||||
req' <- parseRequest $ unpack tarUrl
|
req' <- parseRequest $ unpack tarUrl
|
||||||
let req = req' { decompress = const True }
|
let req = req' { decompress = const True }
|
||||||
|
|
||||||
unlessM (isFile (fromString tarFP)) $ withResponse req man $ \res -> do
|
unlessM (doesFileExist tarFP) $ withResponse req man $ \res -> do
|
||||||
let tmp = tarFP <.> "tmp"
|
let tmp = tarFP <.> "tmp"
|
||||||
createTree $ parent (fromString tmp)
|
createDirectoryIfMissing True $ takeDirectory tmp
|
||||||
runConduitRes
|
runConduitRes
|
||||||
$ bodyReaderSource (responseBody res)
|
$ bodyReaderSource (responseBody res)
|
||||||
.| sinkFile tmp
|
.| sinkFile tmp
|
||||||
rename (fromString tmp) (fromString tarFP)
|
renamePath tmp tarFP
|
||||||
|
|
||||||
void $ tryIO $ removeTree (fromString bindir)
|
void $ tryIO $ removeDirectoryRecursive bindir
|
||||||
void $ tryIO $ removeFile (fromString outname)
|
void $ tryIO $ removeFile outname
|
||||||
createTree (fromString bindir)
|
createDirectoryIfMissing True bindir
|
||||||
|
|
||||||
withSystemTempDirectory ("hoogle-" ++ unpack (toPathPiece name)) $ \tmpdir -> do
|
withSystemTempDirectory ("hoogle-" ++ unpack (toPathPiece name)) $ \tmpdir -> do
|
||||||
allPackagePairs <- runConduitRes
|
allPackagePairs <- runConduitRes
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user