Fineuploader with Grails

Fineuploader is an excellent frontend javascript library for supporting full-featured uploading capabilities such as concurrent file chunking and file resume. Our use case involves uploading extremely large files such as genomic DNA sequencing data including FASTQs, BAMs and VCFs. But the javascript library does run on it's own out of the box. You will still need to implement some server-side code to handle the file uploading.

There's already a Github repository with many examples of server-side implementations for the popular programming languages like Java, Python, PHP, node.js etc... which can be found here:

https://github.com/FineUploader/server-examples

However, I could not find any examples for a Grails implementation, so once again, I rolled up my own solution which can be found below. For this implementation I've only focused on file concurrent chunking and file resume. Other features such as file deletion we're omitted on purpose, but that's not to say you couldn't modify it to support the remaining features of fineuploader.

	package agha.opencga.gui
	import grails.plugin.springsecurity.annotation.Secured
	import groovy.io.FileType
	import org.apache.log4j.Logger
	import org.grails.web.json.JSONObject
	import org.springframework.http.HttpStatus
	import org.springframework.web.multipart.support.StandardMultipartHttpServletRequest
	import javax.servlet.http.HttpServletRequest
	/**
	* An upload controller using fineuploader as the frontend
	*
	* Reference for fineuploader: http://docs.fineuploader.com/branch/master/features/chunking.html
	*
	* @author Philip Wu
	*/
	@Secured(value=["IS_AUTHENTICATED_FULLY"])
	class UploadController {
	Logger logger = Logger.getLogger(UploadController.class)
	//def grailsApplication
	OpenCGAService openCGAService
	def fineuploader() {
	logger.debug('====== fineuploader =====')
	logger.debug('qquuid:'+params.qquuid)
	logger.debug('qqpartindex: '+params.qqpartindex)
	logger.debug('qqpartbyteoffset: '+params.qqpartbyteoffset)
	logger.debug('qqtotalfilesize: '+params.qqpartbyteoffset)
	logger.debug('qqtotalparts: '+params.qqtotalparts)
	logger.debug('qqfilename: '+params.qqfilename)
	logger.debug('qqchunksize: '+params.qqchunksize)
	logger.debug('studyId: '+params.studyId)
	logger.debug('qqresume: '+params.qqresume)
	File tmpFile = writeFile(request, params)
	// If not chunking, move file immediately to proper location once upload has completed
	if (params.qqpartindex == null) {
	File destFolder = createProjectStudyFolder(params.studyId)
	File destFile = new File(destFolder.absolutePath, tmpFile.name)
	// Clean up any previous uploads and overwrite
	if (destFile.exists()) {
	destFile.delete()
	}
	boolean fileMoved = tmpFile.renameTo(destFile)
	logger.debug('fileMoved: '+fileMoved)
	}
	JSONObject json = new JSONObject()
	json.put('success', Boolean.TRUE)
	render(status: HttpStatus.OK, contentType: 'application/json', text: json.toString())
	}
	/**
	* You may specify a chunking.success.endpoint if you'd like your server to be called when all chunks have been successfully uploaded.
	* Note that this only applies to traditional endpoint.
	* @return
	*/
	def fineuploaderChunkSuccess() {
	logger.debug('===== fineuploaderChunkSuccess =====')
	logger.debug('qquuid:'+params.qquuid)
	logger.debug('qqpartindex: '+params.qqpartindex)
	logger.debug('qqpartbyteoffset: '+params.qqpartbyteoffset)
	logger.debug('qqtotalfilesize: '+params.qqpartbyteoffset)
	logger.debug('qqtotalparts: '+params.qqtotalparts)
	logger.debug('qqfilename: '+params.qqfilename)
	logger.debug('qqchunksize: '+params.qqchunksize)
	logger.debug('studyId: '+params.studyId)
	File destFolder = createProjectStudyFolder(params.studyId)
	File destFile = new File(destFolder.absolutePath, params.qqfilename)
	// Clean up any previous uploads and overwrite
	if (destFile.exists()) {
	destFile.delete()
	}
	mergePartitionedFiles(params.qquuid, destFile, params.qqtotalparts.toInteger())
	JSONObject json = new JSONObject()
	json.put('success', Boolean.TRUE)
	render(status: HttpStatus.OK, contentType: 'application/json', text: json.toString())
	}
	private File createTmpFolder(String uuid) {
	String uploadDir = grailsApplication.config.datastore.tmp.path
	File dir = new File(uploadDir, uuid);
	boolean dirMade = dir.mkdirs();
	return dir
	}
	private File createProjectStudyFolder(String studyId) {
	String folder = grailsApplication.config.datastore.path
	// Get the studyInfo
	String sessionId = openCGAService.loginCurrentUser()
	def studyInfo = openCGAService.studyInfo(sessionId, params.studyId)
	// Get the projectId
	String projectId = OpencgaHelper.parseProjectIdFromUri(studyInfo.uri)
	folder += '/'+projectId+'/'+studyId
	File fileFolder = new File(folder)
	fileFolder.mkdirs()
	return fileFolder
	}
	/**
	* Adapted from https://github.com/FineUploader/server-examples/blob/master/java/UploadReceiver.java
	* @param req
	* @param params
	* @throws Exception
	*/
	private File writeFile(HttpServletRequest req, def params) throws Exception
	{
	// Extract needed params
	String uuid = params.qquuid
	Integer partIndex = params.qqpartindex?.toInteger()
	// Create UUID Directory
	File dir = createTmpFolder(uuid)
	logger.debug('dir='+dir)
	logger.debug('content length: '+req.contentLengthLong)
	String filename = (partIndex >= 0) ? createFilePartName(uuid, partIndex) : params.qqfilename
	File destFile = new File(dir, filename)
	FileOutputStream fos = new FileOutputStream(destFile)
	if (req instanceof StandardMultipartHttpServletRequest) {
	req.fileNames.each {
	def mFile = req.getFile(it)
	logger.debug('file: '+mFile)
	fos << mFile.bytes
	}
	} else {
	fos << req.inputStream
	}
	fos.flush()
	return destFile
	}
	private String createFilePartName(String uuid, Integer partIndex) {
	return uuid +'_' + partIndex+ '.part'
	}
	private void mergePartitionedFiles(String uuid, File destFile, Integer totalParts) {
	logger.debug('merging files to: '+destFile)
	// Create UUID Directory
	File dir = createTmpFolder(uuid)
	// Must merge in correct order
	for (int i=0; i < totalParts; i++) {
	String filename = createFilePartName(uuid, i)
	File filePart = new File(dir.absolutePath, filename)
	destFile << filePart.newInputStream()
	// Once appended, delete the part
	boolean deleted = filePart.delete()
	}
	// Delete the temporary folder
	dir.delete()
	}
	}

view raw UploadController hosted with ❤ by GitHub

And for the GSP I have something like this:

	<!doctype html>
	<%@ page import="agha.opencga.gui.*" %>
	<html>
	<head>
	<meta name="layout" content="main"/>
	<title>Study - ${study?.name?.encodeAsHTML() }</title>
	<asset:link rel="icon" href="favicon.ico" type="image/x-ico" />
	<script src="${resource(dir: 'fine-uploader', file:'fine-uploader.js')}" ></script>
	<link rel="stylesheet" href="${resource(dir:'fine-uploader', file:'fine-uploader-new.css') }" />
	</head>
	<body>
	<!-- Fine Uploader Thumbnails template w/ customization
	====================================================================== -->
	<script type="text/template" id="qq-template-manual-trigger">
	<div class="qq-uploader-selector qq-uploader" qq-drop-area-text="Drop files here">
	<div class="qq-total-progress-bar-container-selector qq-total-progress-bar-container">
	<div role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" class="qq-total-progress-bar-selector qq-progress-bar qq-total-progress-bar"></div>
	</div>
	<div class="qq-upload-drop-area-selector qq-upload-drop-area" qq-hide-dropzone>
	<span class="qq-upload-drop-area-text-selector"></span>
	</div>
	<div class="buttons">
	<div class="qq-upload-button-selector qq-upload-button">
	<div>Select files</div>
	</div>
	<button type="button" id="trigger-upload" class="btn btn-primary">
	<i class="icon-upload icon-white"></i> Upload
	</button>
	</div>
	<span class="qq-drop-processing-selector qq-drop-processing">
	<span>Processing dropped files...</span>
	<span class="qq-drop-processing-spinner-selector qq-drop-processing-spinner"></span>
	</span>
	<ul class="qq-upload-list-selector qq-upload-list" aria-live="polite" aria-relevant="additions removals">
	<li>
	<div class="qq-progress-bar-container-selector">
	<div role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" class="qq-progress-bar-selector qq-progress-bar"></div>
	</div>
	<span class="qq-upload-spinner-selector qq-upload-spinner"></span>
	<img class="qq-thumbnail-selector" qq-max-size="100" qq-server-scale>
	<span class="qq-upload-file-selector qq-upload-file"></span>
	<span class="qq-edit-filename-icon-selector qq-edit-filename-icon" aria-label="Edit filename"></span>
	<input class="qq-edit-filename-selector qq-edit-filename" tabindex="0" type="text">
	<span class="qq-upload-size-selector qq-upload-size"></span>
	<button type="button" class="qq-btn qq-upload-cancel-selector qq-upload-cancel">Cancel</button>
	<button type="button" class="qq-btn qq-upload-retry-selector qq-upload-retry">Retry</button>
	<button type="button" class="qq-btn qq-upload-delete-selector qq-upload-delete">Delete</button>
	<span role="status" class="qq-upload-status-text-selector qq-upload-status-text"></span>
	</li>
	</ul>
	<dialog class="qq-alert-dialog-selector">
	<div class="qq-dialog-message-selector"></div>
	<div class="qq-dialog-buttons">
	<button type="button" class="qq-cancel-button-selector">Close</button>
	</div>
	</dialog>
	<dialog class="qq-confirm-dialog-selector">
	<div class="qq-dialog-message-selector"></div>
	<div class="qq-dialog-buttons">
	<button type="button" class="qq-cancel-button-selector">No</button>
	<button type="button" class="qq-ok-button-selector">Yes</button>
	</div>
	</dialog>
	<dialog class="qq-prompt-dialog-selector">
	<div class="qq-dialog-message-selector"></div>
	<input type="text">
	<div class="qq-dialog-buttons">
	<button type="button" class="qq-cancel-button-selector">Cancel</button>
	<button type="button" class="qq-ok-button-selector">Ok</button>
	</div>
	</dialog>
	</div>
	</script>
	<style>
	#trigger-upload {
	color: white;
	background-color: #00ABC7;
	font-size: 14px;
	padding: 7px 20px;
	background-image: none;
	}
	#fine-uploader-manual-trigger .qq-upload-button {
	margin-right: 15px;
	}
	#fine-uploader-manual-trigger .buttons {
	width: 20%;
	margin: 10px;
	padding: 5px;
	}
	#fine-uploader-manual-trigger .qq-uploader .qq-total-progress-bar-container {
	width: 60%;
	}
	</style>
	<div id="content" role="main">
	<section class="row colset-2-its">
	<g:if test="${study}">
	<h1>Study - ${study.name?.encodeAsHTML() }</h1>
	<fieldset>
	<legend>Files</legend>
	<g:if test="${hasFiles}">
	<table>
	<tr>
	<th>Name</th>
	<th>Format</th>
	<th>Size</th>
	</tr>
	<g:each in="${ files }" var="file">
	<g:if test="${file.type == 'FILE'}">
	<tr>
	<td>
	<g:link controller="download" action="download" params="[id: file.id]">
	${file.name?.encodeAsHTML()}
	</g:link>
	</td>
	<td>${file.format?.encodeAsHTML()}</td>
	<td>
	<g:if test="${file.size}">
	${Math.ceil(file.size / 1024 / 1024).toInteger() } MB
	</g:if>
	</td>
	</tr>
	</g:if>
	</g:each>
	</table>
	</g:if>
	<g:else>
	No files available
	</g:else>
	</fieldset>
	<!-- Fine Uploader DOM Element
	====================================================================== -->
	<div id="fine-uploader-manual-trigger"></div>
	</g:if>
	<g:else>
	Study not found
	</g:else>
	</section>
	</div>
	<!-- Your code to create an instance of Fine Uploader and bind to the DOM/template
	====================================================================== -->
	<script>
	var manualUploader = new qq.FineUploader({
	element: document.getElementById('fine-uploader-manual-trigger'),
	template: 'qq-template-manual-trigger',
	request: {
	endpoint: '${createLink(controller:"upload", action:"fineuploader")}',
	params: {studyId: ${study.id} }
	},
	thumbnails: {
	placeholders: {
	waitingPath: '${resource(dir: "fine-uploader/placeholders", file:"waiting-generic.png")}',
	notAvailablePath: '${resource(dir: "fine-uploader/placeholders", file:"not_available-generic.png")}'
	}
	},
	chunking: {
	enabled: true,
	partSize: 50000000,
	concurrent: {
	enabled:true
	},
	success: {
	endpoint: '${createLink(controller:"upload", action:"fineuploaderChunkSuccess")}'
	}
	},
	deleteFile: {
	enabled: true,
	endpoint: '${createLink(controller:"upload", action:"deleteFile")}'
	}
	resume: { enabled: true },
	autoUpload: false,
	debug: true
	});
	qq(document.getElementById("trigger-upload")).attach("click", function() {
	manualUploader.uploadStoredFiles();
	});
	</script>
	</body>
	</html>

view raw upload.gsp hosted with ❤ by GitHub

Gradle intellij

Commands used to get intellij to recognize the Gradle project

gradle idea

From intellij GUI, File -> Invalidate Caches/Restart

Import project from Gradle