fraport/fradrive
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(file-uploads): maximum file sizes

Browse Source
This commit is contained in:
Gregor Kleen 2020-07-13 09:32:28 +02:00
parent 46ce477235
commit 9dee134b11
17 changed files with 198 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
frontend/src/utils/form/form-error-remover.js
View File

@ -36,7 +36,9 @@ export class FormErrorRemover {
inputElements.forEach((inputElement) => {
inputElement.addEventListener('input', () => {
FORM_GROUP_WITH_ERRORS_CLASSES.forEach(c => { this._element.classList.remove(c); });
if (!inputElement.willValidate || inputElement.validity.vaild) {
FORM_GROUP_WITH_ERRORS_CLASSES.forEach(c => { this._element.classList.remove(c); });
}
});
});
}

68
frontend/src/utils/form/form-error-reporter.js Normal file
View File

@ -0,0 +1,68 @@
import { Utility } from '../../core/utility';
import * as defer from 'lodash.defer';
const FORM_ERROR_REPORTER_INITIALIZED_CLASS = 'form-error-remover--initialized';
@Utility({
selector: 'input, textarea, select',
})
export class FormErrorReporter {
_element;
_err;
constructor(element) {
if (!element)
throw new Error('Form Error Reporter utility needs to be passed an element!');
this._element = element;
if (this._element.classList.contains(FORM_ERROR_REPORTER_INITIALIZED_CLASS))
return;
this._element.classList.add(FORM_ERROR_REPORTER_INITIALIZED_CLASS);
}
start() {
if (this._element.willValidate) {
this._element.addEventListener('invalid', this.report.bind(this));
this._element.addEventListener('change', () => { defer(this.report.bind(this)); } );
}
}
report() {
const msg = this._element.validity.valid ? null : this._element.validationMessage;
const target = this._element.closest('.standalone-field, .form-group');
if (!target)
return;
if (this._err && this._err.parentNode) {
this._err.parentNode.removeChild(this._err);
this._err = undefined;
}
if (!msg) {
target.classList.remove('standalone-field--has-error', 'form-group--has-error');
} else {
if (target.classList.contains('form-group')) {
target.classList.add('form-group--has-error');
const container = target.querySelector('.form-group__input');
if (container) {
this._err = document.createElement('div');
this._err.classList.add('form-error');
this._err.innerText = msg;
container.appendChild(this._err);
}
} else {
target.classList.add('standalone-field--has-error');
this._err = document.createElement('div');
this._err.classList.add('standalone-field__error');
this._err.innerText = msg;
target.appendChild(this._err);
}
}
}
}

2
frontend/src/utils/form/form.js
View File

@ -3,6 +3,7 @@ import { AutoSubmitButton } from './auto-submit-button';
import { AutoSubmitInput } from './auto-submit-input';
import { Datepicker } from './datepicker';
import { FormErrorRemover } from './form-error-remover';
import { FormErrorReporter } from './form-error-reporter';
import { InteractiveFieldset } from './interactive-fieldset';
import { NavigateAwayPrompt } from './navigate-away-prompt';
import { CommunicationRecipients } from './communication-recipients';
@ -12,6 +13,7 @@ export const FormUtils = [
AutoSubmitInput,
Datepicker,
FormErrorRemover,
FormErrorReporter,
InteractiveFieldset,
NavigateAwayPrompt,
CommunicationRecipients,

44
frontend/src/utils/inputs/file-max-size.js Normal file
View File

@ -0,0 +1,44 @@
import { Utility } from '../../core/utility';
const FILE_MAX_SIZE_INITIALIZED_CLASS = 'file-max-size--initialized';
@Utility({
selector: 'input[type="file"][data-max-size]',
})
export class FileMaxSize {
_element;
_app;
constructor(element, app) {
if (!element)
throw new Error('FileMaxSize utility cannot be setup without an element!');
this._element = element;
this._app = app;
if (this._element.classList.contains(FILE_MAX_SIZE_INITIALIZED_CLASS)) {
throw new Error('FileMaxSize utility already initialized!');
}
this._element.classList.add(FILE_MAX_SIZE_INITIALIZED_CLASS);
}
start() {
this._element.addEventListener('change', this._change.bind(this));
}
_change() {
const hasOversized = Array.from(this._element.files).some(file => file.size > this._element.dataset.maxSize);
if (hasOversized) {
if (this._element.files.length > 1) {
this._element.setCustomValidity(this._app.i18n.get('fileTooLargeMultiple'));
} else {
this._element.setCustomValidity(this._app.i18n.get('fileTooLarge'));
}
} else {
this._element.setCustomValidity('');
}
this._element.reportValidity();
}
}

2
frontend/src/utils/inputs/inputs.js
View File

@ -1,5 +1,6 @@
import { Checkbox } from './checkbox';
import { FileInput } from './file-input';
import { FileMaxSize } from './file-max-size';
import './inputs.sass';
import './radio-group.sass';
@ -7,4 +8,5 @@ import './radio-group.sass';
export const InputUtils = [
Checkbox,
FileInput,
FileMaxSize,
];

4
messages/frontend/de-de-formal.msg
View File

@ -1,4 +1,6 @@
FilesSelected: Dateien ausgewählt
SelectFile: Datei auswählen
SelectFiles: Datei(en) auswählen
AsyncFormFailure: Da ist etwas schief gelaufen, das tut uns Leid. Falls das erneut passiert schicken Sie uns bitte eine kurze Beschreibung dieses Ereignisses über das Hilfe-Widget rechts oben. Vielen Dank für Ihre Hilfe!
AsyncFormFailure: Da ist etwas schief gelaufen, das tut uns Leid. Falls das erneut passiert schicken Sie uns bitte eine kurze Beschreibung dieses Ereignisses über das Hilfe-Widget rechts oben. Vielen Dank für Ihre Hilfe!
FileTooLarge: Die ausgewählte Datei ist zu groß
FileTooLargeMultiple: Mindestens eine der ausgewählten Dateien ist zu groß

2
messages/frontend/en.msg
View File

@ -2,3 +2,5 @@ FilesSelected: Files selected
SelectFile: Select file
SelectFiles: Select file(s)
AsyncFormFailure: Something went wrong, we are sorry. If this error occurs again, please let us know by clicking the Support button in the upper right corner. Thank you very much!
FileTooLarge: The selected file is too large
FileTooLargeMultiple: At least one of the selected files is too large

5
messages/uniworx/de-de-formal.msg
View File

@ -823,8 +823,11 @@ UploadModeExtensionRestrictionTip: Komma-separiert. Wenn keine Dateiendungen ang
UploadModeExtensionRestrictionEmpty: Liste von zulässigen Dateiendungen darf nicht leer sein
UploadModeExtensionRestrictionMultipleTip: Einschränkung von Dateiendungen erfolgt für alle hochgeladenen Dateien, auch innerhalb von ZIP-Archiven.
GenericFileFieldFileTooLarge file@FilePath: „#{file}“ ist zu groß
GenericFileFieldInvalidExtension file@FilePath: „#{file}” hat keine zulässige Dateiendung
FileUploadOnlySessionTip: Sie haben diese Datei in der aktuellen Session bereits hochgeladen, sie ist allerdings noch nicht gespeichert. Sie müssen zunächst noch das Formular „Senden“, damit die Datei ordnungsgemäß gespeichert wird.
FileUploadMaxSize maxSize@Text: Datei darf maximal #{maxSize} groß sein
FileUploadMaxSizeMultiple maxSize@Text: Dateien dürfen jeweils maximal #{maxSize} groß sein
UploadSpecificFiles: Vorgegebene Dateinamen
NoUploadSpecificFilesConfigured: Wenn der Abgabemodus vorgegebene Dateinamen vorsieht, muss mindestens ein vorgegebener Dateiname konfiguriert werden.
@ -833,6 +836,8 @@ UploadSpecificFilesDuplicateLabels: Bezeichner für vorgegebene Dateinamen müss
UploadSpecificFileLabel: Bezeichnung
UploadSpecificFileName: Dateiname
UploadSpecificFileRequired: Zur Abgabe erforderlich
UploadSpecificFileMaxSize: Maximale Dateigröße (Bytes)
UploadSpecificFileMaxSizeNegative: Maximale Dateigröße darf nicht negativ sein
NoSubmissions: Keine Abgabe
CorrectorSubmissions: Abgabe extern mit Pseudonym

5
messages/uniworx/en-eu.msg
View File

@ -820,8 +820,11 @@ UploadModeExtensionRestrictionTip: Comma-separated. If no file extensions are sp
UploadModeExtensionRestrictionEmpty: List of permitted file extensions may not be empty
UploadModeExtensionRestrictionMultipleTip: Checks for valid file extension are performed for all uploaded files, including those packed within zip-archives.
GenericFileFieldFileTooLarge file: “#{file}” is too large
GenericFileFieldInvalidExtension file: “#{file}” does not have an acceptable file extension
FileUploadOnlySessionTip: You have uploaded this file during your current session. It has not yet been saved permanently. The file will be saved permanently if you “Send” as part of this Form.
FileUploadMaxSize maxSize: File may be up to #{maxSize} in size
FileUploadMaxSizeMultiple maxSize: Files may each be up to #{maxSize} in size
UploadSpecificFiles: Pre-defined files
NoUploadSpecificFilesConfigured: If pre-defined files are selected, at least one file needs to be configured.
@ -830,6 +833,8 @@ UploadSpecificFilesDuplicateLabels: Labels of pre-defined files must be unique
UploadSpecificFileLabel: Label
UploadSpecificFileName: Filename
UploadSpecificFileRequired: Required for submission
UploadSpecificFileMaxSize: Maximum filesize (bytes)
UploadSpecificFileMaxSizeNegative: Maximum filesize may not be negative
NoSubmissions: No submission
CorrectorSubmissions: External submission via pseudonym

67
src/Handler/Utils/Form.hs
View File

@ -30,7 +30,8 @@ import qualified Data.Text as T
import Yesod.Form.Bootstrap3
import Handler.Utils.Zip
import qualified Data.Conduit.List as C
import qualified Data.Conduit.Combinators as C
import qualified Data.Conduit.List as C (mapMaybe, mapMaybeM)
import qualified Database.Esqueleto as E
import qualified Database.Esqueleto.Utils as E
@ -637,9 +638,10 @@ uploadModeForm prev = multiActionA actions (fslI MsgSheetUploadMode) (classifyUp
sFileForm nudge mPrevUF csrf = do
(labelRes, labelView) <- mpreq textField (fslI MsgUploadSpecificFileLabel & addName (nudge "label")) $ specificFileLabel <$> mPrevUF
(nameRes, nameView) <- mpreq textField (fslI MsgUploadSpecificFileName & addName (nudge "name")) $ specificFileName <$> mPrevUF
(maxSizeRes, maxSizeView) <- mopt (natFieldI MsgUploadSpecificFileMaxSizeNegative) (fslI MsgUploadSpecificFileMaxSize & addName (nudge "max-size")) $ specificFileMaxSize <$> mPrevUF
(reqRes, reqView) <- mpreq checkBoxField (fslI MsgUploadSpecificFileRequired & addName (nudge "required")) $ specificFileRequired <$> mPrevUF
return ( UploadSpecificFile <$> labelRes <*> nameRes <*> reqRes
return ( UploadSpecificFile <$> labelRes <*> nameRes <*> reqRes <*> maxSizeRes
, $(widgetFile "widgets/massinput/uploadSpecificFiles/form")
)
@ -846,6 +848,7 @@ data FileField = FileField
, fieldMultiple :: Bool
, fieldRestrictExtensions :: Maybe (NonNull (Set Extension))
, fieldAdditionalFiles :: Map FileId (FileFieldUserOption Bool)
, fieldMaxFileSize :: Maybe Natural -- ^ Applied to each file separately
} deriving (Eq, Ord, Read, Show, Generic, Typeable)
genericFileField :: forall m.
@ -893,23 +896,26 @@ genericFileField mkOpts = Field{..}
$logDebugS "genericFileField.getPermittedFiles" $ "Session: " <> tshow sessionFiles'
return $ fieldAdditionalFiles <> sessionFiles'
handleUpload :: Maybe Text -> File -> DB (Maybe FileId)
handleUpload mIdent file = do
for mIdent $ \ident -> do
now <- liftIO getCurrentTime
oldSFIds <- fmap (Set.fromList . map E.unValue) . E.select . E.from $ \sessionFile -> do
E.where_ $ E.subSelectForeign sessionFile SessionFileFile (E.^. FileTitle) E.==. E.val (fileTitle file)
E.&&. sessionFile E.^. SessionFileTouched E.<=. E.val now
return $ sessionFile E.^. SessionFileId
fId <- insert file
sfId <- insert $ SessionFile fId now
modifySessionJson SessionFiles $ \(fromMaybe mempty -> MergeHashMap old) ->
Just . MergeHashMap $ HashMap.insert ident (Set.insert sfId . maybe Set.empty (`Set.difference` oldSFIds) $ HashMap.lookup ident old) old
return fId
handleUpload :: FileField -> Maybe Text -> File -> DB (Maybe FileId)
handleUpload FileField{fieldMaxFileSize} mIdent file
| maybe (const False) (<) fieldMaxFileSize $ maybe 0 (fromIntegral . olength) (fileContent file)
= return Nothing -- Don't save files that are too large
| otherwise = do
for mIdent $ \ident -> do
now <- liftIO getCurrentTime
oldSFIds <- fmap (Set.fromList . map E.unValue) . E.select . E.from $ \sessionFile -> do
E.where_ $ E.subSelectForeign sessionFile SessionFileFile (E.^. FileTitle) E.==. E.val (fileTitle file)
E.&&. sessionFile E.^. SessionFileTouched E.<=. E.val now
return $ sessionFile E.^. SessionFileId
fId <- insert file
sfId <- insert $ SessionFile fId now
modifySessionJson SessionFiles $ \(fromMaybe mempty -> MergeHashMap old) ->
Just . MergeHashMap $ HashMap.insert ident (Set.insert sfId . maybe Set.empty (`Set.difference` oldSFIds) $ HashMap.lookup ident old) old
return fId
fieldEnctype = Multipart
fieldParse :: [Text] -> [FileInfo] -> m (Either (SomeMessage (HandlerSite m)) (Maybe FileUploads))
fieldParse vals files = do
fieldParse vals files = runExceptT $ do
opts@FileField{..} <- liftHandler mkOpts
mIdent <- fmap getFirst . flip foldMapM vals $ \v ->
@ -933,12 +939,20 @@ genericFileField mkOpts = Field{..}
= not (permittedExtension opts fName)
&& (not doUnpack || ((/=) `on` simpleContentType) (mimeLookup fName) typeZip)
whenIsJust fieldMaxFileSize $ \maxSize -> forM_ files $ \fInfo -> do
fLength <- runConduit $ fileSource fInfo .| C.takeE (fromIntegral $ succ maxSize) .| C.lengthE
when (fLength > maxSize) $ do
liftHandler . runDB . runConduit $
mapM_ (transPipe lift . handleFile) files
.| C.mapM_ (void . handleUpload opts mIdent)
throwE . SomeMessage . MsgGenericFileFieldFileTooLarge . unpack $ fileName fInfo
if | invExt : _ <- filter invalidUploadExtension uploadedFilenames
-> do
liftHandler . runDB . runConduit $
mapM_ (transPipe lift . handleFile) files
.| C.mapM_ (void . handleUpload mIdent)
return . Left . SomeMessage . MsgGenericFileFieldInvalidExtension $ unpack invExt
.| C.mapM_ (void . handleUpload opts mIdent)
throwE . SomeMessage . MsgGenericFileFieldInvalidExtension $ unpack invExt
| otherwise
-> do
let fSrc = do
@ -961,14 +975,14 @@ genericFileField mkOpts = Field{..}
(unsealConduitT -> fSrc', length -> nFiles) <- liftHandler $ fSrc $$+ peekN 2
$logDebugS "genericFileField.fieldParse" $ tshow nFiles
if
| nFiles <= 0 -> return $ Right Nothing
| nFiles <= 1 -> return . Right $ Just fSrc'
| nFiles <= 0 -> return Nothing
| nFiles <= 1 -> return $ Just fSrc'
| not fieldMultiple -> do
liftHandler . runDB . runConduit $
mapM_ (transPipe lift . handleFile) files
.| C.mapM_ (void . handleUpload mIdent)
return . Left $ SomeMessage MsgOnlyUploadOneFile
| otherwise -> return . Right $ Just fSrc'
.| C.mapM_ (void . handleUpload opts mIdent)
throwE $ SomeMessage MsgOnlyUploadOneFile
| otherwise -> return $ Just fSrc'
fieldView :: FieldViewFunc m FileUploads
fieldView fieldId fieldName _attrs val req = do
@ -980,7 +994,7 @@ genericFileField mkOpts = Field{..}
(uploads, references) <- runWriterT . for val $ \src -> do
fmap Set.fromList . sourceToList
$ transPipe (lift . lift) src
.| C.mapMaybeM (either (\fId -> Nothing <$ tell (Set.singleton fId)) $ lift . handleUpload mIdent)
.| C.mapMaybeM (either (\fId -> Nothing <$ tell (Set.singleton fId)) $ lift . handleUpload opts mIdent)
permittedFiles <- getPermittedFiles mIdent opts
@ -1035,6 +1049,7 @@ fileFieldMultiple = genericFileField $ return FileField
, fieldMultiple = True
, fieldRestrictExtensions = Nothing
, fieldAdditionalFiles = Map.empty
, fieldMaxFileSize = Nothing
}
fileField :: (MonadHandler m, HandlerSite m ~ UniWorX) => Field m FileUploads
@ -1044,6 +1059,7 @@ fileField = genericFileField $ return FileField
, fieldMultiple = False
, fieldRestrictExtensions = Nothing
, fieldAdditionalFiles = Map.empty
, fieldMaxFileSize = Nothing
}
specificFileField :: UploadSpecificFile -> Field Handler FileUploads
@ -1053,6 +1069,7 @@ specificFileField UploadSpecificFile{..} = convertField fixupFileTitles id . gen
, fieldMultiple = False
, fieldRestrictExtensions = fromNullable . maybe Set.empty (Set.singleton . view _2) . Map.lookupMin . Map.fromList . map (length &&& id) $ fileNameExtensions specificFileName
, fieldAdditionalFiles = Map.empty
, fieldMaxFileSize = specificFileMaxSize
}
where
fixupFileTitles = flip (.|) . C.mapM $ either (fmap Left . updateFileReference) (fmap Right . updateFile)
@ -1084,6 +1101,7 @@ zipFileField doUnpack permittedExtensions = genericFileField $ return FileField
, fieldMultiple = doUnpack
, fieldRestrictExtensions = permittedExtensions
, fieldAdditionalFiles = Map.empty
, fieldMaxFileSize = Nothing
}
fileUploadForm :: Bool -- ^ Required?
@ -1119,6 +1137,7 @@ multiFileField mkPermitted = genericFileField $ mkField <$> mkPermitted
, fieldMultiple = True
, fieldRestrictExtensions = Nothing
, fieldAdditionalFiles = Map.fromSet (const $ FileFieldUserOption False True) permitted
, fieldMaxFileSize = Nothing
}
data SheetGrading' = Points' | PassPoints' | PassBinary' | PassAlways'

2
src/Model/Types/Sheet.hs
View File

@ -186,10 +186,12 @@ data UploadSpecificFile = UploadSpecificFile
{ specificFileLabel :: Text
, specificFileName :: FileName
, specificFileRequired :: Bool
, specificFileMaxSize :: Maybe Natural
} deriving (Show, Read, Eq, Ord, Generic)
deriveJSON defaultOptions
{ fieldLabelModifier = camelToPathPiece' 2
, omitNothingFields = True
} ''UploadSpecificFile
derivePersistFieldJSON ''UploadSpecificFile

8
src/Utils.hs
View File

@ -258,10 +258,10 @@ textPercent' trailZero precision part whole
-- | Convert number of bytes to human readable format
textBytes :: forall a. Integral a => a -> Text
textBytes x
| v < kb = rshow v <> "B"
| v < mb = rshow (v/kb) <> "KB"
| v < gb = rshow (v/mb) <> "MB"
| otherwise = rshow (v/gb) <> "GB"
| v < kb = rshow v <> "B"
| v < mb = rshow (v/kb) <> "KiB"
| v < gb = rshow (v/mb) <> "MiB"
| otherwise = rshow (v/gb) <> "GiB"
where
v = fromIntegral x
kb :: Double

2
src/Utils/Frontend/I18n.hs
View File

@ -28,6 +28,8 @@ data FrontendMessage = MsgFilesSelected
| MsgSelectFile
| MsgSelectFiles
| MsgAsyncFormFailure
| MsgFileTooLarge
| MsgFileTooLargeMultiple
deriving (Eq, Ord, Enum, Bounded, Read, Show, Generic, Typeable)
instance Universe FrontendMessage
instance Finite FrontendMessage

9
templates/widgets/genericFileField.hamlet
View File

@ -21,7 +21,7 @@ $if not (null fileInfos)
<div .file-uploads-label>_{MsgAddMoreFiles}
$# new files
<input type="file" uw-file-input name=#{fieldName} id=#{fieldId} :fieldMultiple:multiple :acceptRestricted:accept=#{accept} :req && null fileInfos:required>
<input type="file" uw-file-input name=#{fieldName} id=#{fieldId} :fieldMultiple:multiple :acceptRestricted:accept=#{accept} :req && null fileInfos:required :is _Just fieldMaxFileSize:data-max-size=#{maybe "-1" tshow fieldMaxFileSize}>
$if fieldMultiple
<div .file-input__info>
@ -37,6 +37,13 @@ $maybe exts <- fmap toNullable fieldRestrictExtensions
<br>
_{MsgUploadModeExtensionRestrictionMultipleTip}
$maybe maxSize <- fieldMaxFileSize
<div .file-input__info>
$if fieldMultiple
_{MsgFileUploadMaxSizeMultiple (textBytes maxSize)}
$else
_{MsgFileUploadMaxSize (textBytes maxSize)}
$if not (fieldOptionForce fieldUnpackZips)
<div .file-input__unpack>
^{iconTooltip (i18n MsgAutoUnzipInfo) Nothing False}

1
templates/widgets/massinput/uploadSpecificFiles/form.hamlet
View File

@ -1,4 +1,5 @@
$newline never
<td>#{csrf}^{fvWidget labelView}
<td>^{fvWidget nameView}
<td>^{fvWidget maxSizeView}
<td>^{fvWidget reqView}

1
templates/widgets/massinput/uploadSpecificFiles/layout.hamlet
View File

@ -4,6 +4,7 @@ $newline never
<th>_{MsgUploadSpecificFileLabel}
<th>_{MsgUploadSpecificFileName}
<th>_{MsgUploadSpecificFileRequired}
<th>_{MsgUploadSpecificFileMaxSize}
<td>
<tbody>
$forall coord <- review liveCoords lLength

6
test/Database/Fill.hs
View File

@ -757,9 +757,9 @@ fillDb = do
, SubmissionMode corrector $ Just NoUpload
, SubmissionMode corrector $ Just UploadSpecific
{ specificFiles = impureNonNull $ Set.fromList
[ UploadSpecificFile "Aufgabe 1" "exercise_2.1.hs" False
, UploadSpecificFile "Aufgabe 2" "exercise_2.2.hs" False
, UploadSpecificFile "Erklärung der Eigenständigkeit" "erklärung.txt" True
[ UploadSpecificFile "Aufgabe 1" "exercise_2.1.hs" False Nothing
, UploadSpecificFile "Aufgabe 2" "exercise_2.2.hs" False Nothing
, UploadSpecificFile "Erklärung der Eigenständigkeit" "erklärung.txt" True (Just 42)
]
}
] ++ [ SubmissionMode corrector $ Just UploadAny{..}