feat(communication): support attachments in course/tutorial comm's

messages/uniworx/utils/utils/de-de-formal.msg

@@ -17,6 +17,8 @@ RGTutorialParticipants tutn@TutorialName: Tutorium-Teilnehmer:innen (#{tutn})
 RGExamRegistered examn@ExamName: Angemeldet zur Prüfung „#{examn}“
 RGSheetSubmittor shn@SheetName: Abgebende für das Übungsblatt „#{shn}“
 CommSubject: Betreff
+CommAttachments: Anhänge
+CommAttachmentsTip: Im Allgemeinen ist es vorzuziehen Dateien, die Sie mit den Empfängern teilen möchten, als Material hochzuladen (und ggf. in der Nachricht zu verlinken). So ist die Datei für die Empfänger dauerhaft abrufbar und auch Personen, die sich z.B. erst später zum Kurs anmelden, haben Zugriff auf die Datei.
 CommSuccess n@Int: Nachricht wurde an #{n} Empfänger versandt
 CommTestSuccess: Nachricht wurde zu Testzwecken nur an Sie selbst versandt
 

messages/uniworx/utils/utils/en-eu.msg

@@ -17,6 +17,8 @@ RGTutorialParticipants tutn: Tutorial participants (#{tutn})
 RGExamRegistered examn: Registered for exam “#{examn}”
 RGSheetSubmittor shn: Submitted for exercise sheet “#{shn}”
 CommSubject: Subject
+CommAttachments: Attachments
+CommAttachmentsTip: In general it is preferable to upload files as course material instead of sending them as attachments. You can then link to the material from the message. The file is then permanently accessable to the recipients and to persons that, for example, register for the Course at a later date.
 CommSuccess n: Message was sent to #{n} #{pluralEN n "recipient" "recipients"}
 CommTestSuccess: Message was sent only to yourself for testing purposes
 

src/Handler/Utils/Communication.hs

@@ -78,16 +78,16 @@ data CommunicationRoute = CommunicationRoute
 
 data Communication = Communication
  { cRecipients :: Set (Either UserEmail UserId)
- , cSubject :: Maybe Text
+ , cContent :: CommunicationContent
- , cBody :: Html
  }
 
+makeLenses_ ''Communication
+
 
 crJobsCourseCommunication, crTestJobsCourseCommunication :: CourseId -> Communication -> ConduitT () Job (YesodDB UniWorX) ()
 crJobsCourseCommunication jCourse Communication{..} = do
   jSender <- requireAuthId
-  let jSubject = cSubject
+  let jMailContent = cContent
-      jMailContent = cBody
       allRecipients = Set.toList $ Set.insert (Right jSender) cRecipients
   jMailObjectUUID <- liftIO getRandom
   jAllRecipientAddresses <- lift . fmap Set.fromList . forM allRecipients $ \case
@@ -99,7 +99,7 @@ crTestJobsCourseCommunication jCourse comm = do
   jSender <- requireAuthId
 
   MsgRenderer mr <- getMsgRenderer
-  let comm' = comm { cSubject = Just . mr . MsgCommCourseTestSubject . fromMaybe (mr MsgUtilCommCourseSubject) $ cSubject comm }
+  let comm' = comm & _cContent . _ccSubject %~ Just . mr . MsgCommCourseTestSubject . fromMaybe (mr MsgUtilCommCourseSubject)
   crJobsCourseCommunication jCourse comm' .| C.filter ((== Right jSender) . jRecipientEmail)
 
 
@@ -209,8 +209,11 @@ commR CommunicationRoute{..} =  do
   ((commRes,commWdgt),commEncoding) <- runFormPost . identifyForm FIDCommunication . withButtonForm' universeF . renderAForm FormStandard $ Communication
     <$> recipientAForm
     <*  aformMessage recipientsListMsg
-    <*> aopt textField (fslI MsgCommSubject & addAttr "uw-enter-as-tab" "") Nothing
+    <*> ( CommunicationContent
-    <*> (markupOutput <$> areq htmlField (fslI MsgCommBody) Nothing)
+            <$> aopt textField (fslI MsgCommSubject & addAttr "uw-enter-as-tab" "") Nothing
+            <*> (markupOutput <$> areq htmlField (fslI MsgCommBody) Nothing)
+            <*> fmap fold (aopt (convertFieldM (runConduit . (.| C.foldMap Set.singleton)) yieldMany fileFieldMultiple) (fslI MsgCommAttachments & setTooltip MsgCommAttachmentsTip) Nothing)
+        )
   formResult commRes $ \case
     (comm, BtnCommunicationSend) -> do
       runDBJobs . runConduit $ transPipe (mapReaderT lift) (crJobs comm) .| sinkDBJobs

src/Handler/Utils/Files.hs

@@ -1,3 +1,5 @@
+{-# OPTIONS_GHC -fno-warn-orphans #-}
+
 module Handler.Utils.Files
   ( sourceFile, sourceFile'
   , sourceFiles, sourceFiles'
@@ -9,6 +11,7 @@ module Handler.Utils.Files
 
 import Import.NoFoundation hiding (First(..))
 import Foundation.Type
+import Foundation.DB
 import Utils.Metrics
 
 import Data.Monoid (First(..))
@@ -181,6 +184,11 @@ sourceFiles' = C.map sourceFile'
 sourceFile' :: forall file. (HasFileReference file, YesodPersistBackend UniWorX ~ SqlBackend) => file -> DBFile
 sourceFile' = sourceFile . view (_FileReference . _1)
 
+
+instance (YesodMail UniWorX, YesodPersistBackend UniWorX ~ SqlBackend) => ToMailPart UniWorX FileReference where
+  toMailPart = toMailPart <=< liftHandler . runDBRead . withReaderT projectBackend . toPureFile . sourceFile'
+  
+
 respondFileConditional :: (MonadThrow m, MonadHandler m, HandlerSite m ~ UniWorX, YesodPersistBackend UniWorX ~ SqlBackend, YesodPersistRunner UniWorX)
                        => Maybe UTCTime -> MimeType
                        -> FileReference

src/Jobs/Handler/Files.hs

@@ -47,6 +47,9 @@ import qualified Data.Foldable as F
 
 import qualified Control.Monad.State.Class as State
 
+import Jobs.Types
+import Data.Aeson.Lens (_JSON)
+
 
 dispatchJobPruneSessionFiles :: JobHandler UniWorX
 dispatchJobPruneSessionFiles = JobHandlerAtomicWithFinalizer act fin
@@ -83,6 +86,9 @@ workflowFileReferences = Map.fromList $ over (traverse . _1) nameToPathPiece
   , (''WorkflowWorkflow,    E.selectSource (E.from $ pure . (E.^. WorkflowWorkflowState  )) .| awaitForever (mapMOf_ (typesCustom @WorkflowChildren . _fileReferenceContent . _Just) yield . E.unValue))
   ]
 
+jobFileReferences :: MonadResource m => ConduitT () FileContentReference (SqlPersistT m) ()
+jobFileReferences = E.selectSource (E.from $ pure . (E.^. QueuedJobContent)) .| C.mapMaybe (preview _JSON . E.unValue) .| awaitForever (mapMOf_ (typesCustom @JobChildren @Job @Job @FileContentReference @FileContentReference) yield)
+
 
 dispatchJobDetectMissingFiles :: JobHandler UniWorX
 dispatchJobDetectMissingFiles = JobHandlerAtomicDeferrableWithFinalizer act fin
@@ -103,8 +109,10 @@ dispatchJobDetectMissingFiles = JobHandlerAtomicDeferrableWithFinalizer act fin
                 E.distinctOnOrderBy [E.asc ref] $ return ref
           transPipe lift (E.selectSource fileReferencesQuery) .| C.mapMaybe E.unValue .| C.mapM_ (insertRef refKind)
 
-        iforM_ workflowFileReferences $ \refKind refSource ->
+        let useRefSource refKind refSource = transPipe (lift . withReaderT projectBackend) (refSource .| C.filterM (\ref -> not <$> exists [FileContentEntryHash ==. ref])) .| C.mapM_ (insertRef refKind)
-          transPipe (lift . withReaderT projectBackend) (refSource .| C.filterM (\ref -> not <$> exists [FileContentEntryHash ==. ref])) .| C.mapM_ (insertRef refKind)
+        iforM_ workflowFileReferences useRefSource
+        useRefSource (nameToPathPiece ''Job) jobFileReferences
+          
 
       let allMissingDb :: Set Minio.Object
           allMissingDb = setOf (folded . folded . re minioFileReference) missingDb
@@ -204,14 +212,16 @@ dispatchJobPruneUnreferencedFiles numIterations epoch iteration = JobHandlerAtom
           return $ E.any E.exists (fileReferences $ fileContentEntry E.^. FileContentEntryHash)
         E.where_ $ chunkIdFilter unreferencedChunkHash
 
-      let unmarkWorkflowFiles (otoList -> fRefs) = E.delete . E.from $ \fileContentChunkUnreferenced -> do
+      let unmarkSourceFiles (otoList -> fRefs) = E.delete . E.from $ \fileContentChunkUnreferenced -> do
             let unreferencedChunkHash = E.unKey $ fileContentChunkUnreferenced E.^. FileContentChunkUnreferencedHash
             E.where_ . E.subSelectOr . E.from $ \fileContentEntry -> do
               E.where_ $ fileContentEntry E.^. FileContentEntryChunkHash E.==. unreferencedChunkHash
               return $ fileContentEntry E.^. FileContentEntryHash `E.in_` E.valList fRefs 
             E.where_ $ chunkIdFilter unreferencedChunkHash
+          unmarkRefSource refSource = runConduit $ refSource .| C.map Seq.singleton .| C.chunksOfE chunkSize .| C.mapM_ unmarkSourceFiles
           chunkSize = 100
-       in runConduit $ sequence_ workflowFileReferences .| C.map Seq.singleton .| C.chunksOfE chunkSize .| C.mapM_ unmarkWorkflowFiles
+      unmarkRefSource $ sequence_ workflowFileReferences
+      unmarkRefSource   jobFileReferences
   
       let
         getEntryCandidates = E.selectSource . E.from $ \fileContentEntry -> do

src/Jobs/Handler/SendCourseCommunication.hs

@@ -20,10 +20,9 @@ dispatchJobSendCourseCommunication :: Either UserEmail UserId
                                    -> CourseId
                                    -> UserId
                                    -> UUID
-                                   -> Maybe Text
+                                   -> CommunicationContent
-                                   -> Html
                                    -> JobHandler UniWorX
-dispatchJobSendCourseCommunication jRecipientEmail jAllRecipientAddresses jCourse jSender jMailObjectUUID jSubject jMailContent = JobHandlerException $ do
+dispatchJobSendCourseCommunication jRecipientEmail jAllRecipientAddresses jCourse jSender jMailObjectUUID CommunicationContent{..} = JobHandlerException $ do
   (sender, Course{..}) <- runDB $ (,)
     <$> getJust jSender
     <*> getJust jCourse
@@ -34,8 +33,9 @@ dispatchJobSendCourseCommunication jRecipientEmail jAllRecipientAddresses jCours
     _mailFrom .= userAddressFrom sender
     addMailHeader "Cc" [st|#{mr MsgCommUndisclosedRecipients}:;|]
     addMailHeader "Auto-Submitted" "no"
-    setSubjectI . prependCourseTitle courseTerm courseSchool courseShorthand $ maybe (SomeMessage MsgCommCourseSubject) SomeMessage jSubject
+    setSubjectI . prependCourseTitle courseTerm courseSchool courseShorthand $ maybe (SomeMessage MsgCommCourseSubject) SomeMessage ccSubject
     addHtmlMarkdownAlternatives ($(ihamletFile "templates/mail/courseCommunication.hamlet") :: HtmlUrlI18n UniWorXMessage (Route UniWorX))
+    forM_ ccAttachments $ addPart' . toMailPart
     when (jRecipientEmail == Right jSender) $
       addPart' $ do
         partIsAttachmentCsv MsgCommAllRecipients

src/Jobs/Types.hs

@@ -44,7 +44,7 @@ import Cron (CronNextMatch(..), _MatchAsap, _MatchAt, _MatchNone)
 import System.Clock (getTime, Clock(Monotonic), TimeSpec)
 import GHC.Conc (unsafeIOToSTM)
 
-import Data.Generics.Product.Types (Children, ChGeneric)
+import Data.Generics.Product.Types (Children, ChGeneric, HasTypesCustom(..))
 
 {-# ANN module ("HLint: ignore Use newtype instead of data" :: String) #-}
 
@@ -67,8 +67,7 @@ data Job
                                , jCourse :: CourseId
                                , jSender :: UserId
                                , jMailObjectUUID :: UUID
-                               , jSubject :: Maybe Text
+                               , jMailContent :: CommunicationContent
-                               , jMailContent :: Html
                                }
   | JobInvitation { jInviter :: Maybe UserId
                   , jInvitee :: UserEmail
@@ -169,10 +168,14 @@ type family ChildrenJobChildren a where
   ChildrenJobChildren UUID = '[]
   ChildrenJobChildren (Key a) = '[]
   ChildrenJobChildren (CI a) = '[]
-  ChildrenJobChildren (Set a) = '[]
+  ChildrenJobChildren (Set v) = '[v]
   ChildrenJobChildren MailContext = '[]
+  ChildrenJobChildren (Digest a) = '[]
 
   ChildrenJobChildren a = Children ChGeneric a
+
+instance (Ord b', HasTypesCustom JobChildren a' b' a b) => HasTypesCustom JobChildren (Set a') (Set b') a b where
+  typesCustom = iso Set.toList Set.fromList . traverse . typesCustom @JobChildren
   
 
 classifyJob :: Job -> String

src/Mail.hs

@@ -42,6 +42,7 @@ import Data.Kind (Type)
 
 import Model.Types.Languages
 import Model.Types.Csv
+import Model.Types.File
 
 import Network.Mail.Mime hiding (addPart, addAttachment)
 import qualified Network.Mail.Mime as Mime (addPart)
@@ -89,7 +90,7 @@ import qualified Data.Binary as Binary
 import "network-bsd" Network.BSD (getHostName)
 
 import Data.Time.Zones (utcTZ, utcToLocalTimeTZ, timeZoneForUTCTime)
-import Data.Time.LocalTime (ZonedTime(..), TimeZone(..))
+import Data.Time.LocalTime (ZonedTime(..), TimeZone(..), utcToZonedTime, utc)
 import Data.Time.Format (rfc822DateFormat)
 
 import Network.HaskellNet.SMTP (SMTPConnection)
@@ -123,6 +124,12 @@ import Language.Haskell.TH (nameBase)
 
 import Network.Mail.Mime.Instances()
 
+import Data.Char (isLatin1)
+import Data.Text.Lazy.Encoding (decodeUtf8')
+import System.FilePath (takeFileName)
+import Network.HTTP.Types.Header (hETag)
+import Web.HttpApiData (ToHttpApiData(toHeader))
+
 
 makeLenses_ ''Address
 makeLenses_ ''Mail
@@ -346,6 +353,20 @@ instance YesodMail site => ToMailPart site Html where
     _partEncoding .= QuotedPrintableText
     _partContent .= PartContent (renderMarkup html)
 
+instance YesodMail site => ToMailPart site PureFile where
+  toMailPart file@File{fileTitle, fileModified} = do
+    _partDisposition .= AttachmentDisposition (pack $ takeFileName fileTitle)
+    _partType .= decodeUtf8 (mimeLookup $ pack fileTitle)
+    let
+      content :: LBS.ByteString
+      content = file ^. _pureFileContent . _Just
+      isLatin = either (const False) (all isLatin1) $ decodeUtf8' content
+    _partEncoding .= bool Base64 QuotedPrintableText isLatin
+    _partContent .= PartContent content
+    forM_ (file ^. _FileReference . _1 . _fileReferenceContent) $ \fRefContent ->
+      replaceMailHeader (CI.original hETag) . Just . decodeUtf8 . toHeader $ etagFileReference # fRefContent
+    replaceMailHeader (CI.original hLastModified) . Just . pack . formatTime defaultTimeLocale rfc822DateFormat $ utcToZonedTime utc fileModified
+
 instance (ToMailPart site a, RenderMessage site msg) => ToMailPart site (Hamlet.Translate msg -> a) where
   type MailPartReturn site (Hamlet.Translate msg -> a) = MailPartReturn site a
   toMailPart act = do

src/Model/Types.hs

@@ -23,3 +23,4 @@ import Model.Types.Markup as Types
 import Model.Types.Room as Types
 import Model.Types.Csv as Types
 import Model.Types.Upload as Types
+import Model.Types.Communication as Types

src/Model/Types/Communication.hs

@@ -0,0 +1,21 @@
+module Model.Types.Communication
+  ( CommunicationContent(..), _ccSubject, _ccBody, _ccAttachments
+  ) where
+
+import Import.NoModel
+import Model.Types.File
+
+import Utils.Lens.TH
+
+
+data CommunicationContent = CommunicationContent
+  { ccSubject :: Maybe Text
+  , ccBody :: Html
+  , ccAttachments :: Set FileReference
+  } deriving stock (Eq, Ord, Show, Read, Generic, Typeable)
+    deriving anyclass (Hashable, NFData)
+
+deriveJSON  defaultOptions
+  { constructorTagModifier = camelToPathPiece' 1
+  } ''CommunicationContent
+makeLenses_ ''CommunicationContent

src/Model/Types/File.hs

@@ -18,7 +18,24 @@ module Model.Types.File
   , _fieldIdent, _fieldUnpackZips, _fieldMultiple, _fieldRestrictExtensions, _fieldAdditionalFiles, _fieldMaxFileSize
   ) where
 
-import Import.NoModel
+import ClassyPrelude.Yesod hiding (snoc, (.=), getMessageRender, derivePersistFieldJSON, Proxy(..))
+import Crypto.Hash (Digest, SHA3_512)
+import Language.Haskell.TH.Syntax (Lift)
+import Data.Binary (Binary)
+import Crypto.Hash.Instances ()
+import Data.Proxy (Proxy(..))
+import Control.Lens
+import Utils.HttpConditional
+import Data.Binary.Instances.Time ()
+import Data.Time.Clock.Instances ()
+import Data.Aeson.TH
+import Utils
+import Data.Kind (Type)
+import Data.Universe
+import Numeric.Natural
+import Network.Mime
+import Control.Monad.Morph
+import Data.NonNull.Instances ()
    
 import Database.Persist.Sql (PersistFieldSql(..))
 import Web.HttpApiData (ToHttpApiData, FromHttpApiData)
@@ -204,7 +221,6 @@ instance HasFileReference FileReference where
 instance HasFileReference PureFile where
   newtype FileReferenceResidual PureFile = PureFileResidual { unPureFileResidual :: Maybe ByteString }
     deriving (Eq, Ord, Read, Show, Generic, Typeable)
-    deriving newtype (ToJSON, FromJSON)
     deriving anyclass (NFData)
 
   _FileReference = iso toFileReference fromFileReference

templates/mail/courseCommunication.hamlet

@@ -4,4 +4,4 @@ $newline never
   <head>
     <meta charset="UTF-8">
   <body>
-    #{jMailContent}
+    #{ccBody}