Commit bc928091 authored by Alexis SALZMAN's avatar Alexis SALZMAN
Browse files

[xExport] add to GmshMPIIO a file splitting mechanism

A new default parameter is added to xExportGmshMPIIO constructor. This
parameter fix the numbers of files to be generated (see comments in
sources for precise policy).

This addition give a way to split huge results in smaller part more
easily handled with GMSH. It might give also extra performance
for large number of process by enlarging the number of file treated
simultaneously by the file system.
parent 0628714a
......@@ -696,10 +696,62 @@ void xExportGmshMPIIO::clear()
xExportGmshMPIIO::xExportGmshMPIIO(MPI_Comm world) : xExportGmsh(world)
xExportGmshMPIIO::xExportGmshMPIIO(MPI_Comm world, int nb_files) : xExportGmsh(world)
MPI_Comm_rank(world, &proc_id);
MPI_Comm_size(world, &nb_proc);
int file_nb_proc = nb_proc;
int file_proc_id = 0;
// check nb_files
// if less then 2 it is set to 0 which mean one unique file (comunicator is world)
if (nb_files < 2) nb_files = 0;
// if user ask for more then one file and there is at most 7 proc it's request is refused and
// only one file is used
if (nb_files && nb_proc < 8) nb_files = 0;
// set communicator
if (nb_files)
// To force at least 4 proc per file. This avoid user to use MPIIO like Ascii with one file
// per proc. If this one file per proc is wanted better use Ascii/binary version.
int nb_files_limit = nb_proc / 4;
if (nb_files > nb_files_limit) nb_files = nb_files_limit;
// color algo:
// modulo: basic but not clear if it offers best performance
// depend on computer architecture and job managing tool it may be
// the worst case (say inter lam communication instead of intra lam communication)
// auto balanced
// To be tested
// file_proc_id = proc_id % nb_files;
// range: a little more complex but insure, a priori, intra lam communication
// (process are packed on one lam then on the other, ....)
// balanced by use of larger chunk for first files up to shift * (chunck + 1) proc
const int chunk = nb_proc / nb_files;
int shift = nb_proc % nb_files;
int over = 1;
if (shift && proc_id < shift * (chunk + 1))
shift = 0;
over = 0;
file_proc_id = (proc_id - shift) / (chunk + over);
// split comm with colors: 1 color per file
MPI_Comm_split(world, file_proc_id, 0, &univ);
// reset comunicator according to new comunicator
MPI_Comm_rank(univ, &proc_id);
MPI_Comm_size(univ, &nb_proc);
univ = world;
assert(file_nb_proc < 10000);
sprintf(&, "S_%04d_F_%04d_", file_nb_proc, file_proc_id);
MPI_Type_size(MPI_DOUBLE, &szd);
MPI_Type_size(MPI_CHAR, &szc);
MPI_Type_size(MPI_INT, &szi);
......@@ -711,8 +763,8 @@ void xExportGmshMPIIO::openFile(const string &fName)
MPI_Status status;
string loc = fName + getFileNameExtension();
MPI_File_open(world, loc.c_str(), MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &file);
string loc = getFileNamePrefix() + fName + getFileNameExtension();
MPI_File_open(univ, loc.c_str(), MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &file);
MPI_File_set_size(file, 0);
if (!proc_id) MPI_File_write_at(file, 0, const_cast<char *>(head.c_str()), head.size(), MPI_CHAR, &status);
process_started = true;
......@@ -864,12 +916,12 @@ void xExportGmshMPIIO::endView()
MPI_Offset distg[30];
// compute sizes accross all proc
int sizes[29];
MPI_Reduce(nb_info, sizes, 29, MPI_INT, MPI_SUM, last, world);
MPI_Reduce(nb_info, sizes, 29, MPI_INT, MPI_SUM, last, univ);
// compute offset from local size
MPI_Scan(sz_info, offsets, 29, MPI_OFFSET, MPI_SUM, world);
MPI_Scan(sz_info, offsets, 29, MPI_OFFSET, MPI_SUM, univ);
// last proc of world got sum across all proc of number of info to store per information category
// last proc of univ got sum across all proc of number of info to store per information category
if (proc_id == last)
string nbs = "$View\n" + view_name;
......@@ -894,7 +946,7 @@ void xExportGmshMPIIO::endView()
// brodcast distance per information category
MPI_Bcast(distg, 30, MPI_OFFSET, last, world);
MPI_Bcast(distg, 30, MPI_OFFSET, last, univ);
// compute distance per proc per information category
MPI_Offset dist[29];
......@@ -185,10 +185,40 @@ class xExportGmshBinary : public xExportGmsh
tensor_prisms, scalar_pyramids, vector_pyramids, tensor_pyramids, time_step_values;
/*! GMSH export class that use MPIIO functionality: parallel i/o
* The constructor have 2 defaulted arguments:
* - the communicator witch is involved in the export. It could be anything you want. By default it
* - the number of file to generate. By default it is zero witch means that only one file will be created to
* collect all information generated by all world communicator process. A value of one is treated the same way
* as zero. Any other value greater or equal to 2 will be used such as a file is associated to 4 processes
* at least. Thus asking for a large number of files will always be limited by the number of process in the given
* communicator divide by 4:
* nb proc<8 => max number of file is 1
* nb proc=110 => max number of file is 27:
* + If user have set nb_files to 20 then the first 10 files will hold export of 6 processes and
* the 10 last files will hold export of 5 processes. All files use process with ascending id (i.e.
* process from 0 to 5 will be hold by first file, 6 to 11 by second file, ....)
* + If user have set nb_files to 40 then it is reset to 27. The first 2 files will hold export of
* 5 processes and the 25 last files will hold export of 4 processes.
* The constraint imposed by the class that a file must collect information at least from 4 processes avoid using
* MPIIO for to few communication and avoid the case of 1 process witch can be more efficiently treated using
* xExportGmshBinary or xExportGmshAscii.
* User can also fully control the number of files he generates with his export by providing a appropriate communicator,
* setting nb_files and set appropriate file name. For example if using MPI_COMM_SELF then the export will create
* one file per process .... which is not recommended.
* All methods are collective (except addDrawable). Thus if instance of this class are used with Export function, this
* function must be called by all processes of the instance communicator.
class xExportGmshMPIIO : public xExportGmsh
xExportGmshMPIIO(MPI_Comm world = MPI_COMM_WORLD);
xExportGmshMPIIO(MPI_Comm world = MPI_COMM_WORLD, int nb_files = 0);
~xExportGmshMPIIO() override;
void startView(const std::string &comment) override;
void addDrawable(xGmshDrawable::gType, int, double *, double *, double *, double *) override;
......@@ -198,6 +228,7 @@ class xExportGmshMPIIO : public xExportGmsh
void clear();
MPI_Comm univ;
MPI_File file;
std::string view_name, head;
int proc_id, nb_proc, szd, szc, szi;
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment