Hoffman2:MATLAB:qsubjobarray.m
Jump to navigation
Jump to search
%% qsubjobarray.m % % This script uses the Fieldtrip qsub toolbox to submit a job array % that runs the function 'step2_preprocessUpToDipfit' on each individual .set file. % The function performs ASR, AMICA, DIPFIT, and ICLabel which can take about % 3000-14000 seconds to run on a single subject. % % add paths, which will be passed to qsub addpath /u/project/CCN/apps/matlab/fieldtrip/qsub/ % qsub toolbox addpath /u/project/CCN/apps/matlab/eeglab/14.1.0b/ % eeglab % generate list of all .set files within specific folder FILELIST=dir('/u/project/labgroup/EEG/allsubjs/reprocessed/step1/*.set'); % commands to setup env qmem = 2024*2024*2024; % in bytes (~8GB) qtim = 14400; % in seconds (4HR) qopt = '-j y -pe shared 4 '; % options (note trailing space) % usual commands to initiate matlab matlabcmd='. /u/local/Modules/default/init/modules.sh; module load matlab/9.1; matlab'; %% Submit Job Array jobidarray={}; % Enter the main loop for i = 1:size(FILELIST,1) % 500 per user job limit % call to qsubfeval command jobidarray{i}= qsubfeval(@step2_preprocessUpToDipfit, FILELIST(i), ... 'memreq', qmem, ... % bytes 'timreq', qtim, ... % seconds 'backend', 'sge', ... % SGE 'diary', 'always', ... % log level 'options', qopt, ... 'matlabcmd', matlabcmd); pause(10) % 10 seconds (to avoid overload) end % save array to load output at later time save jobarray.mat jobidarray