From 675a9ef3121ec7df56c7495da0efaa42d7cb9da7 Mon Sep 17 00:00:00 2001 From: Adrian Yavorskyy Date: Tue, 7 Feb 2017 16:53:36 +0200 Subject: [PATCH] HUE-2214 [fb] Display and set replication Added displaying of a replication factor in file/folder summary Implemented setting of a replication factor for files Fixed review remarks --- apps/filebrowser/src/filebrowser/forms.py | 5 ++ .../src/filebrowser/templates/listdir.mako | 2 + .../filebrowser/templates/listdir_components.mako | 78 +++++++++++++++++++++- apps/filebrowser/src/filebrowser/urls.py | 1 + apps/filebrowser/src/filebrowser/views.py | 13 +++- desktop/libs/hadoop/src/hadoop/fs/webhdfs.py | 9 +++ 6 files changed, 105 insertions(+), 3 deletions(-) diff --git a/apps/filebrowser/src/filebrowser/forms.py b/apps/filebrowser/src/filebrowser/forms.py index 1c24f57..b24ef0d 100644 --- a/apps/filebrowser/src/filebrowser/forms.py +++ b/apps/filebrowser/src/filebrowser/forms.py @@ -106,6 +106,11 @@ class BaseCopyFormSet(FormSet): CopyFormSet = formset_factory(CopyForm, formset=BaseCopyFormSet, extra=0) +class SetReplicationFactorForm(forms.Form): + op = "setreplication" + src_path = CharField(label=_("File to set replication factor"), help_text=_("The file to set replication factor.")) + replication_factor = CharField(label=_("Value of replication factor"), help_text=_("The value of replication factor.")) + class UploadFileForm(forms.Form): op = "upload" # The "hdfs" prefix in "hdfs_file" triggers the HDFSfileUploadHandler diff --git a/apps/filebrowser/src/filebrowser/templates/listdir.mako b/apps/filebrowser/src/filebrowser/templates/listdir.mako index ff44e17..2eca33b 100644 --- a/apps/filebrowser/src/filebrowser/templates/listdir.mako +++ b/apps/filebrowser/src/filebrowser/templates/listdir.mako @@ -68,6 +68,8 @@ ${ fb_components.menubar() } isCurrentDirSelected().length == 0"> ${_('Move')}
  • ${_('Copy')}
    • +
  • ${_('Set replication factor')}
    • % if show_download_button:
  • diff --git a/apps/filebrowser/src/filebrowser/templates/listdir_components.mako b/apps/filebrowser/src/filebrowser/templates/listdir_components.mako index 2803801..c17b195 100644 --- a/apps/filebrowser/src/filebrowser/templates/listdir_components.mako +++ b/apps/filebrowser/src/filebrowser/templates/listdir_components.mako @@ -237,6 +237,28 @@ from filebrowser.conf import ENABLE_EXTRACT_UPLOADED_ARCHIVE + +
    + ${ csrf_token(request) | n,unicode } + +
    + % if is_fs_superuser:
    @@ -503,6 +525,10 @@ from filebrowser.conf import ENABLE_EXTRACT_UPLOADED_ARCHIVE + ${ _('Replication factor') } + + + ${ _('Number of directories') } @@ -1108,7 +1134,8 @@ from filebrowser.conf import ENABLE_EXTRACT_UPLOADED_ARCHIVE spaceQuota: -1, length: 0, directoryCount: 0, - fileCount: 0 + fileCount: 0, + replication: 0 })); self.showSummary = function () { self.isLoadingSummary(true); @@ -1349,6 +1376,27 @@ from filebrowser.conf import ENABLE_EXTRACT_UPLOADED_ARCHIVE }); }; + self.setReplicationFactor = function () { + $("#SrcPath").attr("value", self.selectedFile().path); + + $("#setReplFileName").text(self.selectedFile().path); + + $("#setReplicationFactorForm").attr("action", "/filebrowser/set_replication_factor?next=${url('filebrowser.views.view', path='')}" + self.currentPath()); + + $('#setReplicationFactorForm').ajaxForm({ + dataType: 'json', + success: function() { + $("#setReplicationModal").modal('hide'); + self.retrieveData(true); + } + }); + + $("#setReplicationModal").modal({ + keyboard:true, + show:true + }); + }; + self.move = function (mode, unselectedDrag) { var paths = []; @@ -1996,6 +2044,20 @@ from filebrowser.conf import ENABLE_EXTRACT_UPLOADED_ARCHIVE hideContextMenu(); }); + // Allow only numeric symbols for setting replication factor + $("#newRepFactorInput").keydown(function (e) { + if ($.inArray(e.keyCode, [46, 8, 9, 27, 13, 110, 190]) !== -1 || + (e.keyCode == 65 && (e.ctrlKey === true || e.metaKey === true)) || + (e.keyCode == 67 && (e.ctrlKey === true || e.metaKey === true)) || + (e.keyCode == 88 && (e.ctrlKey === true || e.metaKey === true)) || + (e.keyCode >= 35 && e.keyCode <= 39)) { + return; + } + if ((e.shiftKey || (e.keyCode < 48 || e.keyCode > 57)) && (e.keyCode < 96 || e.keyCode > 105)) { + e.preventDefault(); + } + }); + // Drag and drop uploads from anywhere on filebrowser screen if (window.FileReader) { var showHoverMsg = function (msg) { @@ -2250,6 +2312,20 @@ from filebrowser.conf import ENABLE_EXTRACT_UPLOADED_ARCHIVE } }); + $("#setReplicationFactorForm").submit(function () { + if ($("#newRepFactorInput").val() == "") { + $("#replicationFactorRequiredAlert").show(); + $("#newRepFactorInput").addClass("fieldError"); + resetPrimaryButtonsStatus(); //globally available + return false; + } + }); + + $("#newRepFactorInput").focus(function () { + $("#replicationFactorRequiredAlert").hide(); + $("#newRepFactorInput").removeClass("fieldError"); + }); + huePubSub.subscribe('update.autocompleters', function(){ $("#moveDestination").jHueHdfsAutocomplete({ showOnFocus: true, diff --git a/apps/filebrowser/src/filebrowser/urls.py b/apps/filebrowser/src/filebrowser/urls.py index 59cb825..e70b325 100644 --- a/apps/filebrowser/src/filebrowser/urls.py +++ b/apps/filebrowser/src/filebrowser/urls.py @@ -47,6 +47,7 @@ urlpatterns = patterns('filebrowser.views', url(r'^touch$', 'touch', name='touch'), url(r'^move$', 'move', name='move'), url(r'^copy$', 'copy', name='copy'), + url(r'^set_replication_factor$', 'set_replication_factor', name='set_replication_factor'), url(r'^rmtree$', 'rmtree', name='rmtree'), url(r'^chmod$', 'chmod', name='chmod'), url(r'^chown$', 'chown', name='chown'), diff --git a/apps/filebrowser/src/filebrowser/views.py b/apps/filebrowser/src/filebrowser/views.py index 3603693..a644338 100644 --- a/apps/filebrowser/src/filebrowser/views.py +++ b/apps/filebrowser/src/filebrowser/views.py @@ -69,7 +69,7 @@ from filebrowser.lib.rwx import filetype, rwx from filebrowser.lib import xxd from filebrowser.forms import RenameForm, UploadFileForm, UploadArchiveForm, MkDirForm, EditorForm, TouchForm,\ RenameFormSet, RmTreeFormSet, ChmodFormSet, ChownFormSet, CopyFormSet, RestoreFormSet,\ - TrashPurgeForm + TrashPurgeForm, SetReplicationFactorForm DEFAULT_CHUNK_SIZE_BYTES = 1024 * 4 # 4KB @@ -528,10 +528,11 @@ def stat(request, path): def content_summary(request, path): if not request.fs.exists(path): raise Http404(_("File not found: %(path)s") % {'path': escape(path)}) - response = {'status': -1, 'message': '', 'summary': None} try: stats = request.fs.get_content_summary(path) + replication_factor = request.fs.stats(path)['replication'] + stats.summary.update({'replication': replication_factor}) response['status'] = 0 response['summary'] = stats.summary except WebHdfsException, e: @@ -1054,6 +1055,14 @@ def rename(request): return generic_op(RenameForm, request, smart_rename, ["src_path", "dest_path"], None) +def set_replication_factor(request): + def smart_set_replication_factor(src_path, replication_factor): + result = request.fs.set_replication_factor(src_path, replication_factor) + if not result: + raise PopupException(_("Setting of replication factor failed")) + + return generic_op(SetReplicationFactorForm, request, smart_set_replication_factor, ["src_path", "replication_factor"], None) + def mkdir(request): def smart_mkdir(path, name): diff --git a/desktop/libs/hadoop/src/hadoop/fs/webhdfs.py b/desktop/libs/hadoop/src/hadoop/fs/webhdfs.py index e45095f..e44fd9e 100644 --- a/desktop/libs/hadoop/src/hadoop/fs/webhdfs.py +++ b/desktop/libs/hadoop/src/hadoop/fs/webhdfs.py @@ -36,6 +36,7 @@ from hadoop.fs.webhdfs_types import WebHdfsStat, WebHdfsContentSummary from hadoop.conf import UPLOAD_CHUNK_SIZE from hadoop.hdfs_site import get_nn_sentry_prefixes, get_umask_mode, get_supergroup + import hadoop.conf import desktop.conf @@ -416,6 +417,14 @@ class WebHdfs(Hdfs): for dirent in ls: self.rename(Hdfs.join(old_dir, dirent), Hdfs.join(new_dir, dirent)) + def set_replication_factor(self, filename, repl_factor): + """set replication factor(filename, repl_factor)""" + params = self._getparams() + params['op'] = 'SETREPLICATION' + params['replication'] = repl_factor + result = self._root.put(filename, params) + return result['boolean'] + def chown(self, path, user=None, group=None, recursive=False): """chown(path, user=None, group=None, recursive=False)""" path = Hdfs.normpath(path) -- 2.9.3 (Apple Git-75)