Hoffman2:MATLAB:qsubjobarray.m

%% qsubjobarray.m
%
%    This script uses the Fieldtrip qsub toolbox to submit a job array
%    that runs the function 'step2_preprocessUpToDipfit' on each individual .set file.
%    

% add paths, which will be passed to qsub
addpath /u/project/CCN/apps/matlab/fieldtrip/qsub/    % qsub toolbox
addpath /u/project/CCN/apps/matlab/eeglab/14.1.0b/    % eeglab 

% generate list of all .set files within specific folder
FILELIST=dir('/u/project/labgroup/EEG/allsubjs/reprocessed/step1/*.set');

% commands to setup env
qmem = 2024*2024*2024;       % in bytes (~8GB)
qtim = 14400;                % in seconds (4HR)
qopt = '-j y -pe shared 4 '; % options (note trailing space)

% usual commands to initiate matlab
matlabcmd='. /u/local/Modules/default/init/modules.sh; module load matlab/9.1; matlab';


%% Submit Job Array 
jobidarray={};

% Enter the main loop
for i = 1:size(FILELIST,1) % 500 per user job limit 
  
  % call to qsubfeval command
  jobidarray{i}= qsubfeval(@step2_preprocessUpToDipfit, FILELIST(i), ...
                       'memreq',    qmem,     ... % bytes
                       'timreq',    qtim,     ... % seconds 
                       'backend',   'sge',    ... % SGE
                       'diary',     'always', ... % log level
                       'options',   qopt,     ...
                       'matlabcmd', matlabcmd);
  
  pause(10) % 10 seconds (to avoid overload)

end

% save array to load output at later time
save jobarray.mat jobidarray