Prevent duplicate files + fixing first upload

- server now warns if file is duplicate (below 1G)
- fixing upload status on the first file of the bundle

.env.example

@@ -9,6 +9,8 @@ APP_LOCALE=en
 UPLOAD_MAX_FILESIZE=1G
 UPLOAD_MAX_FILES=1000
 UPLOAD_LIMIT_IPS=127.0.0.1
+UPLOAD_PREVENT_DUPLICATES=true
+HASH_MAX_FILESIZE=1G
 
 FILESYSTEM_DISK=local
 SESSION_DRIVER=file

app/Helpers/Upload.php

@@ -45,7 +45,7 @@ class Upload {
 			];
 		}
 
-		array_push($metadata['files'], $file);
+		array_unshift($metadata['files'], $file);
 		self::setMetadata($bundleId, $metadata);
 
 		return $metadata;
@@ -93,12 +93,7 @@ class Upload {
 		];
 
 		foreach ($values as $k => $v) {
-			$unit = preg_replace('/[^bkmgtpezy]/i', '', $v);
+			$values[$k] = self::humanReadableToBytes($v);
-			$size = preg_replace('/[^0-9\.]/', '', $v);
-			if ($unit) {
-				// Find the position of the unit in the ordered string which is the power of magnitude to multiply a kilobyte by.
-				$values[$k] = round($size * pow(1024, stripos('bkmgtpezy', $unit[0])), 1);
-			}
 		}
 
 		$min = min($values);
@@ -108,6 +103,25 @@ class Upload {
 		return $min;
 	}
 
+	public static function humanReadableToBytes($value) {
+		$unit = preg_replace('/[^bkmgtpezy]/i', '', $value);
+		$size = preg_replace('/[^0-9\.]/', '', $value);
+		if (! empty($unit)) {
+			$value = round($size * pow(1024, stripos('bkmgtpezy', $unit[0])), 1);
+		}
+		return $value;
+	}
+
+	public static function isDuplicateFile($bundleId, $hash) {
+		$metadata = self::getMetadata($bundleId);
+		foreach ($metadata['files'] as $f) {
+			if ($f['hash'] !== null && $f['hash'] == $hash) {
+				return true;
+			}
+		}
+		return false;
+	}
+
 	public static function canUpload($current_ip) {
 
 		// Getting the IP limit configuration

app/Http/Controllers/UploadController.php

@@ -63,15 +63,25 @@ class UploadController extends Controller
 				$bundleId, $filename, 'uploads'
 			);
 
+			$size = Storage::disk('uploads')->size($fullpath);
+			if (config('sharing.upload_prevent_duplicates', true) === true && $size < Upload::humanReadableToBytes(config('sharing.hash_maxfilesize', '1G'))) {
+				$hash = sha1_file(Storage::disk('uploads')->path($fullpath));
+				if (Upload::isDuplicateFile($bundleId, $hash)) {
+					Storage::disk('uploads')->delete($fullpath);
+					throw new Exception(__('app.duplicate-file'));
+				}
+			}
+
 			// Generating file metadata
 			$file = [
 				'uuid'  				=> $request->uuid,
 				'original'  			=> $original,
-				'filesize'  			=> Storage::disk('uploads')->size($fullpath),
+				'filesize'  			=> $size,
 				'fullpath'  			=> $fullpath,
 				'filename'				=> $filename,
 				'created_at'			=> time(),
-				'status'				=> true
+				'status'				=> true,
+				'hash'					=> $hash ?? null
 			];
 
 			$metadata = Upload::addFileMetaData($bundleId, $file);

config/sharing.php

@@ -29,4 +29,11 @@ return [
      */
     'upload_ip_limit'		=> env('UPLOAD_LIMIT_IPS', null),
 
+	'upload_prevent_duplicates'	=> env('UPLOAD_PREVENT_DUPLICATES', true),
+
+	/**
+	 ** Max filesize hash processing
+	 ** TODO: find the best value to avoid too long time processing
+	 */
+	 'hash_maxfilesize'		=> env('HASH_MAX_FILESIZE', '1G')
 ];

lang/en/app.php

@@ -81,4 +81,5 @@ return [
 	'password' => 'Password',
 	'do-login' => 'Login now',
 	'pending'	=> 'Drafts',
+	'duplicate-file' => 'This file already exists in the bundle'
 ];

lang/fr/app.php

@@ -81,4 +81,5 @@ return [
 	'password' => 'Mot de passe',
 	'do-login' => 'S\'authentifier',
 	'pending' => 'Brouillons',
+	'duplicate-file' => 'Ce fichier existe déjà dans l\'archive'
 ];

readme.md

@@ -89,6 +89,8 @@ In order to configure your application, copy the .env.example file into .env. Th
 | `APP_DEBUG` | change this to `false` when in production (`true` otherwise) |
 | `APP_TIMEZONE` | change this to your current timezone |
 | `APP_LOCALE` | change this to "fr" or "en" |
+| `UPLOAD_PREVENT_DUPLICATES` | Should the app block duplicate files (true | false) |
+| `HASH_MAX_FILESIZE`| max filesize for hashing file to check for duplicate files. If files are higher, they will not be hashed. Find the best value to better time / memory consumption |
 | `UPLOAD_MAX_FILES` | (*optional*) maximal number of files per bundle |
 | `UPLOAD_MAX_FILESIZE` | (*optional*) change this to the value you want (K, M, G, T, ...). Attention : you must configure your PHP settings too (`post_max_size`, `upload_max_filesize` and `memory_limit`). When missing, using PHP lowest configuration |
 | `UPLOAD_LIMIT_IPS` | (*optional*) a comma separated list of IPs from which you may upload files. Different formats are supported : Full IP address (192.168.10.2), Wildcard format (192.168.10.*), CIDR Format (192.168.10/24 or 1.2.3.4/255.255.255.0) or Start-end IP (192.168.10.0-192.168.10.10). When missing, filtering is disabled. |

resources/views/upload.blade.php

@@ -202,11 +202,9 @@
 					})
 
 					this.dropzone.on('addedfile', (file) => {
-						console.log('added file')
-
 						file.uuid = this.uuid()
 
-						this.metadata.files.push({
+						this.metadata.files.unshift({
 							uuid: file.uuid,
 							original: file.name,
 							filesize: file.size,
@@ -215,6 +213,7 @@
 							created_at: moment().unix(),
 							status: 'uploading'
 						});
+
 					})
 
 					this.dropzone.on('sending', (file, xhr, data) => {
@@ -224,15 +223,21 @@
 					this.dropzone.on('uploadprogress', (file, progress, bytes) => {
 						let fileIndex = null
 
-						if (fileIndex = this.findFileIndex(file.upload.uuid)) {
+						if (fileIndex = this.findFileIndex(file.uuid)) {
 							this.metadata.files[fileIndex].progress = Math.round(progress)
 						}
 					})
 
 					this.dropzone.on('error', (file, message) => {
-						let fileIndex = this.findFileIndex(file.upload.uuid)
+						let fileIndex = this.findFileIndex(file.uuid)
 						this.metadata.files[fileIndex].status = false
+
+						if (message.hasOwnProperty('error')) {
+							this.metadata.files[fileIndex].message = message.error
+						}
+						else {
 							this.metadata.files[fileIndex].message = message
+						}
 					})
 
 					this.dropzone.on('complete', (file) => {
@@ -606,10 +611,10 @@
 									</div>
 								</h3>
 
-								<span class="text-xs text-slate-400" x-show="countFilesOnServer() == 0">@lang('app.no-file')</span>
+								<span class="text-xs text-slate-400" x-show="Object.keys(metadata.files).length == 0">@lang('app.no-file')</span>
 
 								{{-- Files list --}}
-								<ul id="output" class="text-xs max-h-32 overflow-y-scroll pb-3" x-show="countFilesOnServer() > 0">
+								<ul id="output" class="text-xs max-h-32 overflow-y-scroll pb-3" x-show="Object.keys(metadata.files).length > 0">
 									<template x-for="(f, k) in metadata.files" :key="k">
 										<li
 											title="{{ __('app.click-to-remove') }}"