diff --git a/messages/uniworx/utils/utils/de-de-formal.msg b/messages/uniworx/utils/utils/de-de-formal.msg index 1464f36ae..82a4e02f3 100644 --- a/messages/uniworx/utils/utils/de-de-formal.msg +++ b/messages/uniworx/utils/utils/de-de-formal.msg @@ -17,6 +17,8 @@ RGTutorialParticipants tutn@TutorialName: Tutorium-Teilnehmer:innen (#{tutn}) RGExamRegistered examn@ExamName: Angemeldet zur Prüfung „#{examn}“ RGSheetSubmittor shn@SheetName: Abgebende für das Übungsblatt „#{shn}“ CommSubject: Betreff +CommAttachments: Anhänge +CommAttachmentsTip: Im Allgemeinen ist es vorzuziehen Dateien, die Sie mit den Empfängern teilen möchten, als Material hochzuladen (und ggf. in der Nachricht zu verlinken). So ist die Datei für die Empfänger dauerhaft abrufbar und auch Personen, die sich z.B. erst später zum Kurs anmelden, haben Zugriff auf die Datei. CommSuccess n@Int: Nachricht wurde an #{n} Empfänger versandt CommTestSuccess: Nachricht wurde zu Testzwecken nur an Sie selbst versandt diff --git a/messages/uniworx/utils/utils/en-eu.msg b/messages/uniworx/utils/utils/en-eu.msg index 1539fdf4c..28a834e93 100644 --- a/messages/uniworx/utils/utils/en-eu.msg +++ b/messages/uniworx/utils/utils/en-eu.msg @@ -17,6 +17,8 @@ RGTutorialParticipants tutn: Tutorial participants (#{tutn}) RGExamRegistered examn: Registered for exam “#{examn}” RGSheetSubmittor shn: Submitted for exercise sheet “#{shn}” CommSubject: Subject +CommAttachments: Attachments +CommAttachmentsTip: In general it is preferable to upload files as course material instead of sending them as attachments. You can then link to the material from the message. The file is then permanently accessable to the recipients and to persons that, for example, register for the Course at a later date. CommSuccess n: Message was sent to #{n} #{pluralEN n "recipient" "recipients"} CommTestSuccess: Message was sent only to yourself for testing purposes diff --git a/src/Handler/Utils/Communication.hs b/src/Handler/Utils/Communication.hs index ca32a1b71..4cfca1a04 100644 --- a/src/Handler/Utils/Communication.hs +++ b/src/Handler/Utils/Communication.hs @@ -78,16 +78,16 @@ data CommunicationRoute = CommunicationRoute data Communication = Communication { cRecipients :: Set (Either UserEmail UserId) - , cSubject :: Maybe Text - , cBody :: Html + , cContent :: CommunicationContent } +makeLenses_ ''Communication + crJobsCourseCommunication, crTestJobsCourseCommunication :: CourseId -> Communication -> ConduitT () Job (YesodDB UniWorX) () crJobsCourseCommunication jCourse Communication{..} = do jSender <- requireAuthId - let jSubject = cSubject - jMailContent = cBody + let jMailContent = cContent allRecipients = Set.toList $ Set.insert (Right jSender) cRecipients jMailObjectUUID <- liftIO getRandom jAllRecipientAddresses <- lift . fmap Set.fromList . forM allRecipients $ \case @@ -99,7 +99,7 @@ crTestJobsCourseCommunication jCourse comm = do jSender <- requireAuthId MsgRenderer mr <- getMsgRenderer - let comm' = comm { cSubject = Just . mr . MsgCommCourseTestSubject . fromMaybe (mr MsgUtilCommCourseSubject) $ cSubject comm } + let comm' = comm & _cContent . _ccSubject %~ Just . mr . MsgCommCourseTestSubject . fromMaybe (mr MsgUtilCommCourseSubject) crJobsCourseCommunication jCourse comm' .| C.filter ((== Right jSender) . jRecipientEmail) @@ -209,8 +209,11 @@ commR CommunicationRoute{..} = do ((commRes,commWdgt),commEncoding) <- runFormPost . identifyForm FIDCommunication . withButtonForm' universeF . renderAForm FormStandard $ Communication <$> recipientAForm <* aformMessage recipientsListMsg - <*> aopt textField (fslI MsgCommSubject & addAttr "uw-enter-as-tab" "") Nothing - <*> (markupOutput <$> areq htmlField (fslI MsgCommBody) Nothing) + <*> ( CommunicationContent + <$> aopt textField (fslI MsgCommSubject & addAttr "uw-enter-as-tab" "") Nothing + <*> (markupOutput <$> areq htmlField (fslI MsgCommBody) Nothing) + <*> fmap fold (aopt (convertFieldM (runConduit . (.| C.foldMap Set.singleton)) yieldMany fileFieldMultiple) (fslI MsgCommAttachments & setTooltip MsgCommAttachmentsTip) Nothing) + ) formResult commRes $ \case (comm, BtnCommunicationSend) -> do runDBJobs . runConduit $ transPipe (mapReaderT lift) (crJobs comm) .| sinkDBJobs diff --git a/src/Handler/Utils/Files.hs b/src/Handler/Utils/Files.hs index 83b5f7552..98b1e602e 100644 --- a/src/Handler/Utils/Files.hs +++ b/src/Handler/Utils/Files.hs @@ -1,3 +1,5 @@ +{-# OPTIONS_GHC -fno-warn-orphans #-} + module Handler.Utils.Files ( sourceFile, sourceFile' , sourceFiles, sourceFiles' @@ -9,6 +11,7 @@ module Handler.Utils.Files import Import.NoFoundation hiding (First(..)) import Foundation.Type +import Foundation.DB import Utils.Metrics import Data.Monoid (First(..)) @@ -181,6 +184,11 @@ sourceFiles' = C.map sourceFile' sourceFile' :: forall file. (HasFileReference file, YesodPersistBackend UniWorX ~ SqlBackend) => file -> DBFile sourceFile' = sourceFile . view (_FileReference . _1) + +instance (YesodMail UniWorX, YesodPersistBackend UniWorX ~ SqlBackend) => ToMailPart UniWorX FileReference where + toMailPart = toMailPart <=< liftHandler . runDBRead . withReaderT projectBackend . toPureFile . sourceFile' + + respondFileConditional :: (MonadThrow m, MonadHandler m, HandlerSite m ~ UniWorX, YesodPersistBackend UniWorX ~ SqlBackend, YesodPersistRunner UniWorX) => Maybe UTCTime -> MimeType -> FileReference diff --git a/src/Jobs/Handler/Files.hs b/src/Jobs/Handler/Files.hs index 7ab592eb1..de6787c0d 100644 --- a/src/Jobs/Handler/Files.hs +++ b/src/Jobs/Handler/Files.hs @@ -47,6 +47,9 @@ import qualified Data.Foldable as F import qualified Control.Monad.State.Class as State +import Jobs.Types +import Data.Aeson.Lens (_JSON) + dispatchJobPruneSessionFiles :: JobHandler UniWorX dispatchJobPruneSessionFiles = JobHandlerAtomicWithFinalizer act fin @@ -83,6 +86,9 @@ workflowFileReferences = Map.fromList $ over (traverse . _1) nameToPathPiece , (''WorkflowWorkflow, E.selectSource (E.from $ pure . (E.^. WorkflowWorkflowState )) .| awaitForever (mapMOf_ (typesCustom @WorkflowChildren . _fileReferenceContent . _Just) yield . E.unValue)) ] +jobFileReferences :: MonadResource m => ConduitT () FileContentReference (SqlPersistT m) () +jobFileReferences = E.selectSource (E.from $ pure . (E.^. QueuedJobContent)) .| C.mapMaybe (preview _JSON . E.unValue) .| awaitForever (mapMOf_ (typesCustom @JobChildren @Job @Job @FileContentReference @FileContentReference) yield) + dispatchJobDetectMissingFiles :: JobHandler UniWorX dispatchJobDetectMissingFiles = JobHandlerAtomicDeferrableWithFinalizer act fin @@ -103,8 +109,10 @@ dispatchJobDetectMissingFiles = JobHandlerAtomicDeferrableWithFinalizer act fin E.distinctOnOrderBy [E.asc ref] $ return ref transPipe lift (E.selectSource fileReferencesQuery) .| C.mapMaybe E.unValue .| C.mapM_ (insertRef refKind) - iforM_ workflowFileReferences $ \refKind refSource -> - transPipe (lift . withReaderT projectBackend) (refSource .| C.filterM (\ref -> not <$> exists [FileContentEntryHash ==. ref])) .| C.mapM_ (insertRef refKind) + let useRefSource refKind refSource = transPipe (lift . withReaderT projectBackend) (refSource .| C.filterM (\ref -> not <$> exists [FileContentEntryHash ==. ref])) .| C.mapM_ (insertRef refKind) + iforM_ workflowFileReferences useRefSource + useRefSource (nameToPathPiece ''Job) jobFileReferences + let allMissingDb :: Set Minio.Object allMissingDb = setOf (folded . folded . re minioFileReference) missingDb @@ -204,14 +212,16 @@ dispatchJobPruneUnreferencedFiles numIterations epoch iteration = JobHandlerAtom return $ E.any E.exists (fileReferences $ fileContentEntry E.^. FileContentEntryHash) E.where_ $ chunkIdFilter unreferencedChunkHash - let unmarkWorkflowFiles (otoList -> fRefs) = E.delete . E.from $ \fileContentChunkUnreferenced -> do + let unmarkSourceFiles (otoList -> fRefs) = E.delete . E.from $ \fileContentChunkUnreferenced -> do let unreferencedChunkHash = E.unKey $ fileContentChunkUnreferenced E.^. FileContentChunkUnreferencedHash E.where_ . E.subSelectOr . E.from $ \fileContentEntry -> do E.where_ $ fileContentEntry E.^. FileContentEntryChunkHash E.==. unreferencedChunkHash return $ fileContentEntry E.^. FileContentEntryHash `E.in_` E.valList fRefs E.where_ $ chunkIdFilter unreferencedChunkHash + unmarkRefSource refSource = runConduit $ refSource .| C.map Seq.singleton .| C.chunksOfE chunkSize .| C.mapM_ unmarkSourceFiles chunkSize = 100 - in runConduit $ sequence_ workflowFileReferences .| C.map Seq.singleton .| C.chunksOfE chunkSize .| C.mapM_ unmarkWorkflowFiles + unmarkRefSource $ sequence_ workflowFileReferences + unmarkRefSource jobFileReferences let getEntryCandidates = E.selectSource . E.from $ \fileContentEntry -> do diff --git a/src/Jobs/Handler/SendCourseCommunication.hs b/src/Jobs/Handler/SendCourseCommunication.hs index 712fd4beb..7a3433645 100644 --- a/src/Jobs/Handler/SendCourseCommunication.hs +++ b/src/Jobs/Handler/SendCourseCommunication.hs @@ -20,10 +20,9 @@ dispatchJobSendCourseCommunication :: Either UserEmail UserId -> CourseId -> UserId -> UUID - -> Maybe Text - -> Html + -> CommunicationContent -> JobHandler UniWorX -dispatchJobSendCourseCommunication jRecipientEmail jAllRecipientAddresses jCourse jSender jMailObjectUUID jSubject jMailContent = JobHandlerException $ do +dispatchJobSendCourseCommunication jRecipientEmail jAllRecipientAddresses jCourse jSender jMailObjectUUID CommunicationContent{..} = JobHandlerException $ do (sender, Course{..}) <- runDB $ (,) <$> getJust jSender <*> getJust jCourse @@ -34,8 +33,9 @@ dispatchJobSendCourseCommunication jRecipientEmail jAllRecipientAddresses jCours _mailFrom .= userAddressFrom sender addMailHeader "Cc" [st|#{mr MsgCommUndisclosedRecipients}:;|] addMailHeader "Auto-Submitted" "no" - setSubjectI . prependCourseTitle courseTerm courseSchool courseShorthand $ maybe (SomeMessage MsgCommCourseSubject) SomeMessage jSubject + setSubjectI . prependCourseTitle courseTerm courseSchool courseShorthand $ maybe (SomeMessage MsgCommCourseSubject) SomeMessage ccSubject addHtmlMarkdownAlternatives ($(ihamletFile "templates/mail/courseCommunication.hamlet") :: HtmlUrlI18n UniWorXMessage (Route UniWorX)) + forM_ ccAttachments $ addPart' . toMailPart when (jRecipientEmail == Right jSender) $ addPart' $ do partIsAttachmentCsv MsgCommAllRecipients diff --git a/src/Jobs/Types.hs b/src/Jobs/Types.hs index 94afb6b53..9efc5df8c 100644 --- a/src/Jobs/Types.hs +++ b/src/Jobs/Types.hs @@ -44,7 +44,7 @@ import Cron (CronNextMatch(..), _MatchAsap, _MatchAt, _MatchNone) import System.Clock (getTime, Clock(Monotonic), TimeSpec) import GHC.Conc (unsafeIOToSTM) -import Data.Generics.Product.Types (Children, ChGeneric) +import Data.Generics.Product.Types (Children, ChGeneric, HasTypesCustom(..)) {-# ANN module ("HLint: ignore Use newtype instead of data" :: String) #-} @@ -67,8 +67,7 @@ data Job , jCourse :: CourseId , jSender :: UserId , jMailObjectUUID :: UUID - , jSubject :: Maybe Text - , jMailContent :: Html + , jMailContent :: CommunicationContent } | JobInvitation { jInviter :: Maybe UserId , jInvitee :: UserEmail @@ -169,10 +168,14 @@ type family ChildrenJobChildren a where ChildrenJobChildren UUID = '[] ChildrenJobChildren (Key a) = '[] ChildrenJobChildren (CI a) = '[] - ChildrenJobChildren (Set a) = '[] + ChildrenJobChildren (Set v) = '[v] ChildrenJobChildren MailContext = '[] + ChildrenJobChildren (Digest a) = '[] ChildrenJobChildren a = Children ChGeneric a + +instance (Ord b', HasTypesCustom JobChildren a' b' a b) => HasTypesCustom JobChildren (Set a') (Set b') a b where + typesCustom = iso Set.toList Set.fromList . traverse . typesCustom @JobChildren classifyJob :: Job -> String diff --git a/src/Mail.hs b/src/Mail.hs index 827467b8e..4a3d560fb 100644 --- a/src/Mail.hs +++ b/src/Mail.hs @@ -42,6 +42,7 @@ import Data.Kind (Type) import Model.Types.Languages import Model.Types.Csv +import Model.Types.File import Network.Mail.Mime hiding (addPart, addAttachment) import qualified Network.Mail.Mime as Mime (addPart) @@ -89,7 +90,7 @@ import qualified Data.Binary as Binary import "network-bsd" Network.BSD (getHostName) import Data.Time.Zones (utcTZ, utcToLocalTimeTZ, timeZoneForUTCTime) -import Data.Time.LocalTime (ZonedTime(..), TimeZone(..)) +import Data.Time.LocalTime (ZonedTime(..), TimeZone(..), utcToZonedTime, utc) import Data.Time.Format (rfc822DateFormat) import Network.HaskellNet.SMTP (SMTPConnection) @@ -123,6 +124,12 @@ import Language.Haskell.TH (nameBase) import Network.Mail.Mime.Instances() +import Data.Char (isLatin1) +import Data.Text.Lazy.Encoding (decodeUtf8') +import System.FilePath (takeFileName) +import Network.HTTP.Types.Header (hETag) +import Web.HttpApiData (ToHttpApiData(toHeader)) + makeLenses_ ''Address makeLenses_ ''Mail @@ -346,6 +353,20 @@ instance YesodMail site => ToMailPart site Html where _partEncoding .= QuotedPrintableText _partContent .= PartContent (renderMarkup html) +instance YesodMail site => ToMailPart site PureFile where + toMailPart file@File{fileTitle, fileModified} = do + _partDisposition .= AttachmentDisposition (pack $ takeFileName fileTitle) + _partType .= decodeUtf8 (mimeLookup $ pack fileTitle) + let + content :: LBS.ByteString + content = file ^. _pureFileContent . _Just + isLatin = either (const False) (all isLatin1) $ decodeUtf8' content + _partEncoding .= bool Base64 QuotedPrintableText isLatin + _partContent .= PartContent content + forM_ (file ^. _FileReference . _1 . _fileReferenceContent) $ \fRefContent -> + replaceMailHeader (CI.original hETag) . Just . decodeUtf8 . toHeader $ etagFileReference # fRefContent + replaceMailHeader (CI.original hLastModified) . Just . pack . formatTime defaultTimeLocale rfc822DateFormat $ utcToZonedTime utc fileModified + instance (ToMailPart site a, RenderMessage site msg) => ToMailPart site (Hamlet.Translate msg -> a) where type MailPartReturn site (Hamlet.Translate msg -> a) = MailPartReturn site a toMailPart act = do diff --git a/src/Model/Types.hs b/src/Model/Types.hs index 5b5562675..ac591631c 100644 --- a/src/Model/Types.hs +++ b/src/Model/Types.hs @@ -23,3 +23,4 @@ import Model.Types.Markup as Types import Model.Types.Room as Types import Model.Types.Csv as Types import Model.Types.Upload as Types +import Model.Types.Communication as Types diff --git a/src/Model/Types/Communication.hs b/src/Model/Types/Communication.hs new file mode 100644 index 000000000..b21f3e101 --- /dev/null +++ b/src/Model/Types/Communication.hs @@ -0,0 +1,21 @@ +module Model.Types.Communication + ( CommunicationContent(..), _ccSubject, _ccBody, _ccAttachments + ) where + +import Import.NoModel +import Model.Types.File + +import Utils.Lens.TH + + +data CommunicationContent = CommunicationContent + { ccSubject :: Maybe Text + , ccBody :: Html + , ccAttachments :: Set FileReference + } deriving stock (Eq, Ord, Show, Read, Generic, Typeable) + deriving anyclass (Hashable, NFData) + +deriveJSON defaultOptions + { constructorTagModifier = camelToPathPiece' 1 + } ''CommunicationContent +makeLenses_ ''CommunicationContent diff --git a/src/Model/Types/File.hs b/src/Model/Types/File.hs index 0a3819c28..fae0b9a0c 100644 --- a/src/Model/Types/File.hs +++ b/src/Model/Types/File.hs @@ -18,7 +18,24 @@ module Model.Types.File , _fieldIdent, _fieldUnpackZips, _fieldMultiple, _fieldRestrictExtensions, _fieldAdditionalFiles, _fieldMaxFileSize ) where -import Import.NoModel +import ClassyPrelude.Yesod hiding (snoc, (.=), getMessageRender, derivePersistFieldJSON, Proxy(..)) +import Crypto.Hash (Digest, SHA3_512) +import Language.Haskell.TH.Syntax (Lift) +import Data.Binary (Binary) +import Crypto.Hash.Instances () +import Data.Proxy (Proxy(..)) +import Control.Lens +import Utils.HttpConditional +import Data.Binary.Instances.Time () +import Data.Time.Clock.Instances () +import Data.Aeson.TH +import Utils +import Data.Kind (Type) +import Data.Universe +import Numeric.Natural +import Network.Mime +import Control.Monad.Morph +import Data.NonNull.Instances () import Database.Persist.Sql (PersistFieldSql(..)) import Web.HttpApiData (ToHttpApiData, FromHttpApiData) @@ -204,7 +221,6 @@ instance HasFileReference FileReference where instance HasFileReference PureFile where newtype FileReferenceResidual PureFile = PureFileResidual { unPureFileResidual :: Maybe ByteString } deriving (Eq, Ord, Read, Show, Generic, Typeable) - deriving newtype (ToJSON, FromJSON) deriving anyclass (NFData) _FileReference = iso toFileReference fromFileReference diff --git a/templates/mail/courseCommunication.hamlet b/templates/mail/courseCommunication.hamlet index b63e2827e..b6e305827 100644 --- a/templates/mail/courseCommunication.hamlet +++ b/templates/mail/courseCommunication.hamlet @@ -4,4 +4,4 @@ $newline never - #{jMailContent} + #{ccBody}