Skip to content

GitLab

Commit c8a5bf8b authored by Gaurav Saxena's avatar Gaurav Saxena
Browse files

BioFVM with X decomposition - obtaining much better timings than the Z... 

BioFVM with X decomposition - obtaining much better timings than the Z decomposition. Also remember to use the ppr notation in the script file. Best results being obtained from 2 processes per node i.e. one per NUMA region and then nodes can be increased.
parent 975859a6
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 576 additions and 424 deletions
BioFVM_microenvironment.cpp
View file @ c8a5bf8b
......@@ -778,7 +778,7 @@ void Microenvironment::simulate_bulk_sources_and_sinks( double dt )
void Microenvironment::simulate_cell_sources_and_sinks( std::vector<Basic_Agent*>& basic_agent_list , double dt )
{
//#pragma omp parallel for
#pragma omp parallel for
for( int i=0 ; i < basic_agent_list.size() ; i++ )
{
basic_agent_list[i]->simulate_secretion_and_uptake( this , dt );
......
BioFVM_parallel.cpp
View file @ c8a5bf8b
This diff is collapsed.
Makefile
View file @ c8a5bf8b
......@@ -2,8 +2,15 @@ VERSION := 1.1.6
PROGRAM_NAME := make_lib
#For GCC
CC := mpic++
#For Intel
#CC := mpiicpc
#For TAU
#CC := tau_cxx.sh
# change this to your own CPU archicture.
# Here is a list for gcc 4.9.0
# https://gcc.gnu.org/onlinedocs/gcc-4.9.0/gcc/i386-and-x86-64-Options.html#i386-and-x86-64-Options
......@@ -24,7 +31,10 @@ ARCH := native # best auto-tuning
# ARCH := silvermont
# ARCH := nocona #64-bit pentium 4 or later
CFLAGS := -march=$(ARCH) -O3 -fomit-frame-pointer -mfpmath=both -fopenmp -m64 -std=c++11 -g
#First for GCC and second for ICPC
CFLAGS := -march=$(ARCH) -O2 -fomit-frame-pointer -mfpmath=both -fopenmp -m64 -std=c++11 -g
#CFLAGS := -march=$(ARCH) -O2 -fomit-frame-pointer -qopenmp -m64 -std=c++11 -g
BioFVM_OBJECTS := BioFVM_vector.o BioFVM_matlab.o BioFVM_utilities.o BioFVM_mesh.o \
BioFVM_microenvironment.o BioFVM_solvers.o BioFVM_basic_agent.o \
......
error-7584197 0 → 100644
View file @ c8a5bf8b
[s06r1b59:192760] MCW rank 0 bound to socket 0[core 0[hwt 0]], socket 0[core 1[hwt 0]], socket 0[core 2[hwt 0]], socket 0[core 3[hwt 0]], socket 0[core 4[hwt 0]], socket 0[core 5[hwt 0]], socket 0[core 6[hwt 0]], socket 0[core 7[hwt 0]], socket 0[core 8[hwt 0]], socket 0[core 9[hwt 0]], socket 0[core 10[hwt 0]], socket 0[core 11[hwt 0]], socket 0[core 12[hwt 0]], socket 0[core 13[hwt 0]], socket 0[core 14[hwt 0]], socket 0[core 15[hwt 0]], socket 0[core 16[hwt 0]], socket 0[core 17[hwt 0]], socket 0[core 18[hwt 0]], socket 0[core 19[hwt 0]], socket 0[core 20[hwt 0]], socket 0[core 21[hwt 0]], socket 0[core 22[hwt 0]], socket 0[core 23[hwt 0]]: [B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B][./././././././././././././././././././././././.]
[s06r1b59:192760] MCW rank 1 bound to socket 1[core 24[hwt 0]], socket 1[core 25[hwt 0]], socket 1[core 26[hwt 0]], socket 1[core 27[hwt 0]], socket 1[core 28[hwt 0]], socket 1[core 29[hwt 0]], socket 1[core 30[hwt 0]], socket 1[core 31[hwt 0]], socket 1[core 32[hwt 0]], socket 1[core 33[hwt 0]], socket 1[core 34[hwt 0]], socket 1[core 35[hwt 0]], socket 1[core 36[hwt 0]], socket 1[core 37[hwt 0]], socket 1[core 38[hwt 0]], socket 1[core 39[hwt 0]], socket 1[core 40[hwt 0]], socket 1[core 41[hwt 0]], socket 1[core 42[hwt 0]], socket 1[core 43[hwt 0]], socket 1[core 44[hwt 0]], socket 1[core 45[hwt 0]], socket 1[core 46[hwt 0]], socket 1[core 47[hwt 0]]: [./././././././././././././././././././././././.][B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B]
[s06r1b61:272153] MCW rank 2 bound to socket 0[core 0[hwt 0]], socket 0[core 1[hwt 0]], socket 0[core 2[hwt 0]], socket 0[core 3[hwt 0]], socket 0[core 4[hwt 0]], socket 0[core 5[hwt 0]], socket 0[core 6[hwt 0]], socket 0[core 7[hwt 0]], socket 0[core 8[hwt 0]], socket 0[core 9[hwt 0]], socket 0[core 10[hwt 0]], socket 0[core 11[hwt 0]], socket 0[core 12[hwt 0]], socket 0[core 13[hwt 0]], socket 0[core 14[hwt 0]], socket 0[core 15[hwt 0]], socket 0[core 16[hwt 0]], socket 0[core 17[hwt 0]], socket 0[core 18[hwt 0]], socket 0[core 19[hwt 0]], socket 0[core 20[hwt 0]], socket 0[core 21[hwt 0]], socket 0[core 22[hwt 0]], socket 0[core 23[hwt 0]]: [B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B][./././././././././././././././././././././././.]
[s06r1b65:389755] MCW rank 6 bound to socket 0[core 0[hwt 0]], socket 0[core 1[hwt 0]], socket 0[core 2[hwt 0]], socket 0[core 3[hwt 0]], socket 0[core 4[hwt 0]], socket 0[core 5[hwt 0]], socket 0[core 6[hwt 0]], socket 0[core 7[hwt 0]], socket 0[core 8[hwt 0]], socket 0[core 9[hwt 0]], socket 0[core 10[hwt 0]], socket 0[core 11[hwt 0]], socket 0[core 12[hwt 0]], socket 0[core 13[hwt 0]], socket 0[core 14[hwt 0]], socket 0[core 15[hwt 0]], socket 0[core 16[hwt 0]], socket 0[core 17[hwt 0]], socket 0[core 18[hwt 0]], socket 0[core 19[hwt 0]], socket 0[core 20[hwt 0]], socket 0[core 21[hwt 0]], socket 0[core 22[hwt 0]], socket 0[core 23[hwt 0]]: [B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B][./././././././././././././././././././././././.]
[s06r1b61:272153] MCW rank 3 bound to socket 1[core 24[hwt 0]], socket 1[core 25[hwt 0]], socket 1[core 26[hwt 0]], socket 1[core 27[hwt 0]], socket 1[core 28[hwt 0]], socket 1[core 29[hwt 0]], socket 1[core 30[hwt 0]], socket 1[core 31[hwt 0]], socket 1[core 32[hwt 0]], socket 1[core 33[hwt 0]], socket 1[core 34[hwt 0]], socket 1[core 35[hwt 0]], socket 1[core 36[hwt 0]], socket 1[core 37[hwt 0]], socket 1[core 38[hwt 0]], socket 1[core 39[hwt 0]], socket 1[core 40[hwt 0]], socket 1[core 41[hwt 0]], socket 1[core 42[hwt 0]], socket 1[core 43[hwt 0]], socket 1[core 44[hwt 0]], socket 1[core 45[hwt 0]], socket 1[core 46[hwt 0]], socket 1[core 47[hwt 0]]: [./././././././././././././././././././././././.][B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B]
[s06r1b65:389755] MCW rank 7 bound to socket 1[core 24[hwt 0]], socket 1[core 25[hwt 0]], socket 1[core 26[hwt 0]], socket 1[core 27[hwt 0]], socket 1[core 28[hwt 0]], socket 1[core 29[hwt 0]], socket 1[core 30[hwt 0]], socket 1[core 31[hwt 0]], socket 1[core 32[hwt 0]], socket 1[core 33[hwt 0]], socket 1[core 34[hwt 0]], socket 1[core 35[hwt 0]], socket 1[core 36[hwt 0]], socket 1[core 37[hwt 0]], socket 1[core 38[hwt 0]], socket 1[core 39[hwt 0]], socket 1[core 40[hwt 0]], socket 1[core 41[hwt 0]], socket 1[core 42[hwt 0]], socket 1[core 43[hwt 0]], socket 1[core 44[hwt 0]], socket 1[core 45[hwt 0]], socket 1[core 46[hwt 0]], socket 1[core 47[hwt 0]]: [./././././././././././././././././././././././.][B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B]
[s06r1b64:351888] MCW rank 4 bound to socket 0[core 0[hwt 0]], socket 0[core 1[hwt 0]], socket 0[core 2[hwt 0]], socket 0[core 3[hwt 0]], socket 0[core 4[hwt 0]], socket 0[core 5[hwt 0]], socket 0[core 6[hwt 0]], socket 0[core 7[hwt 0]], socket 0[core 8[hwt 0]], socket 0[core 9[hwt 0]], socket 0[core 10[hwt 0]], socket 0[core 11[hwt 0]], socket 0[core 12[hwt 0]], socket 0[core 13[hwt 0]], socket 0[core 14[hwt 0]], socket 0[core 15[hwt 0]], socket 0[core 16[hwt 0]], socket 0[core 17[hwt 0]], socket 0[core 18[hwt 0]], socket 0[core 19[hwt 0]], socket 0[core 20[hwt 0]], socket 0[core 21[hwt 0]], socket 0[core 22[hwt 0]], socket 0[core 23[hwt 0]]: [B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B][./././././././././././././././././././././././.]
[s06r1b64:351888] MCW rank 5 bound to socket 1[core 24[hwt 0]], socket 1[core 25[hwt 0]], socket 1[core 26[hwt 0]], socket 1[core 27[hwt 0]], socket 1[core 28[hwt 0]], socket 1[core 29[hwt 0]], socket 1[core 30[hwt 0]], socket 1[core 31[hwt 0]], socket 1[core 32[hwt 0]], socket 1[core 33[hwt 0]], socket 1[core 34[hwt 0]], socket 1[core 35[hwt 0]], socket 1[core 36[hwt 0]], socket 1[core 37[hwt 0]], socket 1[core 38[hwt 0]], socket 1[core 39[hwt 0]], socket 1[core 40[hwt 0]], socket 1[core 41[hwt 0]], socket 1[core 42[hwt 0]], socket 1[core 43[hwt 0]], socket 1[core 44[hwt 0]], socket 1[core 45[hwt 0]], socket 1[core 46[hwt 0]], socket 1[core 47[hwt 0]]: [./././././././././././././././././././././././.][B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B/B]
OPENMP DISPLAY ENVIRONMENT BEGIN
_OPENMP = '201511'
OPENMP DISPLAY ENVIRONMENT BEGIN
_OPENMP = '201511'
OMP_DYNAMIC = 'FALSE'
OMP_NESTED = 'FALSE'
OMP_NUM_THREADS = '24'
OMP_SCHEDULE = 'DYNAMIC'
OMP_PROC_BIND = 'SPREAD'
OMP_PLACES = '{ OMP_DYNAMIC = 'FALSE'
OMP_NESTED = 'FALSE'
OMP_NUM_THREADS = '24'
OMP_SCHEDULE = 'DYNAMIC'
OMP_PROC_BIND = 'SPREAD'
OMP_PLACES = '{024},{25},{26},{27},{28},{29},{30},{},{1},{2},{3},{4},{5},{6},{31},{32},{33},{34},{35},{36},{37},{38},{7},{8},{9},{10},{11},{12},{13},39},{40},{41},{42},{43},{44},{45},{46},{47}{14},{15},{16},{17},{18},{19},{20},{21},{22},'
OMP_STACKSIZE = '0'
OMP_WAIT_POLICY = 'PASSIVE'
OMP_THREAD_LIMIT = '4294967295'
OMP_MAX_ACTIVE_LEVELS = '2147483647'
OMP_CANCELLATION = 'FALSE'
OMP_DEFAULT_DEVICE = '0'
OMP_MAX_TASK_PRIORITY = '0'
OPENMP DISPLAY ENVIRONMENT END
{23}'
OMP_STACKSIZE = '0'
OMP_WAIT_POLICY = 'PASSIVE'
OMP_THREAD_LIMIT = '4294967295'
OMP_MAX_ACTIVE_LEVELS = '2147483647'
OMP_CANCELLATION = 'FALSE'
OMP_DEFAULT_DEVICE = '0'
OMP_MAX_TASK_PRIORITY = '0'
OPENMP DISPLAY ENVIRONMENT END
OPENMP DISPLAY ENVIRONMENT BEGIN
_OPENMP = '201511'
OMP_DYNAMIC = 'FALSE'
OMP_NESTED = 'FALSE'
OMP_NUM_THREADS = '24'
OMP_SCHEDULE = 'DYNAMIC'
OMP_PROC_BIND = 'SPREAD'
OMP_PLACES = '{0},{1},{2},{3},{4},{5},{6},{
OPENMP DISPLAY ENVIRONMENT BEGIN
_OPENMP = '201511'
OMP_DYNAMIC = 'FALSE'
7},{8},{9},{10},{11},{
OPENMP DISPLAY ENVIRONMENT BEGIN
_OPENMP = '201511'
OMP_DYNAMIC = 'FALSE'
OMP_NESTED = 'FALSE'
OMP_NUM_THREADS = '24'
OMP_SCHEDULE = 'DYNAMIC'
OMP_PROC_BIND = 'SPREAD'
OMP_PLACES = '{12},{13},{14},{15},{16},{ OMP_NESTED = 'FALSE'
OMP_NUM_THREADS = '24'
OMP_SCHEDULE = 'DYNAMIC'
OMP_PROC_BIND = 'SPREAD'
OMP_PLACES = '{17},{18},{19},{20},{21},{0},{1},{222},{23}'
OMP_STACKSIZE = '0'
OMP_WAIT_POLICY = 'PASSIVE'
OMP_THREAD_LIMIT = '4294967295'
OMP_MAX_ACTIVE_LEVELS = '2147483647'
OMP_CANCELLATION = 'FALSE'
},{3},{4},{5},{6 OMP_DEFAULT_DEVICE = '0'
OMP_MAX_TASK_PRIORITY = '0'
OPENMP DISPLAY ENVIRONMENT END
24},{25},{26},{27},{28
OPENMP DISPLAY ENVIRONMENT BEGIN
_OPENMP = '201511'
OMP_DYNAMIC = 'FALSE'
OMP_NESTED = 'FALSE'
OMP_NUM_THREADS = '24'
OMP_SCHEDULE = 'DYNAMIC'
OMP_PROC_BIND = 'SPREAD'
OMP_PLACES = '{24},{25},{26},{27},{28},{29},{30},{31},{32},{33},{34},{35},{36},{37},{38},{39},{40},{41},{42},{43},{44},{45},{},{7},{8},{9},{10},{11},{12},{1346},{47}'
OMP_STACKSIZE = '0'
OMP_WAIT_POLICY = 'PASSIVE'
OMP_THREAD_LIMIT = '4294967295'
OMP_MAX_ACTIVE_LEVELS = '2147483647'
OMP_CANCELLATION = 'FALSE'
OMP_DEFAULT_DEVICE = '0'
OMP_MAX_TASK_PRIORITY = '0'
OPENMP DISPLAY ENVIRONMENT END
},{29},{30},{31},{32},{33},{34},{35},{36},},{14},{15},{16},{17},{18},{19},{20},{21},{22},{23}'
OMP_STACKSIZE = '0'
OMP_WAIT_POLICY = 'PASSIVE'
OMP_THREAD_LIMIT = '4294967295'
OMP_MAX_ACTIVE_LEVELS = '2147483647'
OMP_CANCELLATION = 'FALSE'
OMP_DEFAULT_DEVICE = '0'
{37},{38},{39},{40},{41},{42},{43},{44},{45},{46},{47}'
OMP_STACKSIZE = '0'
OMP_WAIT_POLICY = 'PASSIVE'
OMP_THREAD_LIMIT = '4294967295'
OMP_MAX_ACTIVE_LEVELS = '2147483647'
OMP_CANCELLATION = 'FALSE'
OMP_DEFAULT_DEVICE = '0'
OMP_MAX_TASK_PRIORITY = '0'
OPENMP DISPLAY ENVIRONMENT END
OMP_MAX_TASK_PRIORITY = '0'
OPENMP DISPLAY ENVIRONMENT END
OPENMP DISPLAY ENVIRONMENT BEGIN
_OPENMP = '201511'
OMP_DYNAMIC = 'FALSE'
OMP_NESTED = 'FALSE'
OMP_NUM_THREADS = '24'
OMP_SCHEDULE = 'DYNAMIC'
OMP_PROC_BIND = 'SPREAD'
OMP_PLACES = '{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14},{15},{16},
OPENMP DISPLAY ENVIRONMENT BEGIN
_OPENMP = '201511'
OMP_DYNAMIC = 'FALSE'
OMP_NESTED = 'FALSE'
OMP_NUM_THREADS = '24'
OMP_SCHEDULE = 'DYNAMIC'
OMP_PROC_BIND = 'SPREAD'
OMP_PLACES = '{24},{25},{26},{27{17},{18},{19},{20},{21},{22},{23}'
OMP_STACKSIZE = '0'
OMP_WAIT_POLICY = 'PASSIVE'
OMP_THREAD_LIMIT = '4294967295'
OMP_MAX_ACTIVE_LEVELS = '2147483647'
OMP_CANCELLATION = 'FALSE'
OMP_DEFAULT_DEVICE = '0'
OMP_MAX_TASK_PRIORITY = '0'
OPENMP DISPLAY ENVIRONMENT END
},{28},{29},{30},{31},{32},{33},{34},{35},{36},{37},{38},{39},{40},{41},{42},{43},{44},{45},{46},{47}'
OMP_STACKSIZE = '0'
OMP_WAIT_POLICY = 'PASSIVE'
OMP_THREAD_LIMIT = '4294967295'
OMP_MAX_ACTIVE_LEVELS = '2147483647'
OMP_CANCELLATION = 'FALSE'
OMP_DEFAULT_DEVICE = '0'
OMP_MAX_TASK_PRIORITY = '0'
OPENMP DISPLAY ENVIRONMENT END
examples/tutorial1
View file @ c8a5bf8b
No preview for this file type
examples/tutorial1_BioFVM.cpp
View file @ c8a5bf8b
......@@ -55,6 +55,7 @@
#include <fstream>
#include <random>
/*----------------Include MPI Header-------------*/
#include <mpi.h>
/*----------------------------------------------*/
......@@ -64,7 +65,7 @@
using namespace BioFVM;
int omp_num_threads = 1; // set number of threads for parallel computing, set this to # of CPU cores x 2 (for hyperthreading)
//int omp_num_threads = 48; // set number of threads for parallel computing, set this to # of CPU cores x 2 (for hyperthreading)
......@@ -222,10 +223,10 @@ void create_point_sources(double cell_radius, double dt, int num_sources, Microe
/* this class is also generating an ID, and we do not want the constructor ID */
/*------------------------------------------------------------------------------------------------*/
for(int r=0; r<mpi_Size; r++)
{
if(mpi_Rank == r)
{
//for(int r=0; r<mpi_Size; r++)
//{
// if(mpi_Rank == r)
// {
for(int k=0 ; k<sources_per_proc; k++)
{
Basic_Agent *temp_point_source = create_basic_agent();
......@@ -236,15 +237,15 @@ void create_point_sources(double cell_radius, double dt, int num_sources, Microe
tempPoint[i] = list_of_coords[3*k + i];
temp_point_source->assign_position(tempPoint, mpi_Rank, mpi_Dims);
temp_point_source->ID = list_of_IDs[k]; //Added this statement to overwrite Constructor generated ID.
std::cout <<"SOURCE:" << "Rank = "<< mpi_Rank << "(" << tempPoint[0] << "," << tempPoint[1] << "," << tempPoint[2] << ")" << "ID = " << list_of_IDs[k]<<std::endl;
//std::cout <<"SOURCE:" << "Rank = "<< mpi_Rank << "(" << tempPoint[0] << "," << tempPoint[1] << "," << tempPoint[2] << ")" << "ID = " << list_of_IDs[k]<<std::endl;
temp_point_source->set_total_volume( (4.0/3.0)*pi*pow(cell_radius,3.0) );
(*temp_point_source->secretion_rates)[0]=10;
(*temp_point_source->saturation_densities)[0]=1;
temp_point_source->set_internal_uptake_constants(dt);
}
}
MPI_Barrier(mpi_Cart_comm);
}
// }
// MPI_Barrier(mpi_Cart_comm);
//}
}
......@@ -388,39 +389,41 @@ void create_point_sinks(double cell_radius, double dt, int num_sinks, Microenvir
/* this class is also generating an ID, and we do not want the constructor ID */
/*------------------------------------------------------------------------------------------------*/
for(int r = 0; r< mpi_Size; r++)
{
if(mpi_Rank == r)
{
//for(int r = 0; r< mpi_Size; r++)
//{
// if(mpi_Rank == r)
// {
for(int k=0 ; k<sinks_per_proc; k++)
{
Basic_Agent *temp_point_sink = create_basic_agent();
Basic_Agent *temp_point_sink = create_basic_agent();
temp_point_sink->register_microenvironment(&microenvironment);
temp_point_sink->register_microenvironment(&microenvironment);
for(int i=0 ; i<3 ; i++)
tempPoint[i] = list_of_coords[3*k + i];
temp_point_sink->assign_position(tempPoint, mpi_Rank, mpi_Dims);
std::cout << "SINK:" << "Rank = "<< mpi_Rank << "(" << tempPoint[0] << "," << tempPoint[1] << "," << tempPoint[2] << ")" <<"ID = " << list_of_IDs[k] << std::endl;
temp_point_sink->ID = list_of_IDs[k]; //Added this statement to overwrite Constructor generated ID.
temp_point_sink->set_total_volume( (4.0/3.0)*pi*pow(cell_radius,3.0) );
(*temp_point_sink->uptake_rates)[0]=0.8;
temp_point_sink->set_internal_uptake_constants(dt);
}
}
MPI_Barrier(mpi_Cart_comm);
}
for(int i=0 ; i<3 ; i++)
tempPoint[i] = list_of_coords[3*k + i];
temp_point_sink->assign_position(tempPoint, mpi_Rank, mpi_Dims);
//std::cout << "SINK:" << "Rank = "<< mpi_Rank << "(" << tempPoint[0] << "," << tempPoint[1] << "," << tempPoint[2] << ")" <<"ID = " << list_of_IDs[k] << std::endl;
temp_point_sink->ID = list_of_IDs[k]; //Added this statement to overwrite Constructor generated ID.
temp_point_sink->set_total_volume( (4.0/3.0)*pi*pow(cell_radius,3.0) );
(*temp_point_sink->uptake_rates)[0]=0.8;
temp_point_sink->set_internal_uptake_constants(dt);
}
// }
// MPI_Barrier(mpi_Cart_comm);
//}
}
int main( int argc, char* argv[] )
{
double minX=0, minY=0, minZ=0, maxX=1000, maxY=1000, maxZ=1000, mesh_resolution=10;
double minX=0, minY=0, minZ=0, maxX=1440, maxY=1440, maxZ=1440, mesh_resolution=10;
/*--------------------------------------------*/
/* Initialize MPI */
/*--------------------------------------------*/
double t_start, t_end, t_total_start, t_total_end;
unsigned int mpi_Error;
int mpi_Requested_level = MPI_THREAD_FUNNELED;
int mpi_Provided_level;
......@@ -430,8 +433,11 @@ int main( int argc, char* argv[] )
int mpi_Dims[3], mpi_Is_periodic[3], mpi_Coords[3];
int mpi_Reorder;
mpi_Error = MPI_Init_thread(NULL, NULL, mpi_Requested_level, &mpi_Provided_level);
mpi_Error = MPI_Init_thread(NULL, NULL, mpi_Requested_level, &mpi_Provided_level);
MPI_Barrier(mpi_Comm);
t_total_start = MPI_Wtime();
if(mpi_Error != MPI_SUCCESS)
MPI_Abort(mpi_Comm,-1);
......@@ -451,23 +457,23 @@ int main( int argc, char* argv[] )
mpi_Dims[0] = 1;
mpi_Dims[1] = 1;
mpi_Dims[2] = mpi_Size;
//MPI_Dims_create(mpi_Size,3,mpi_Dims);
mpi_Dims[1] = mpi_Size; //Number of Y processes, since Y-processes divide X-axis, we have X-decomposition
mpi_Dims[2] = 1;
//MPI_Dims_create(mpi_Size,3,mpi_Dims); //-->no need as we already know the decomposition i.e. 1-D Z-decomposition
mpi_Is_periodic[0]=0;
mpi_Is_periodic[1]=0;
mpi_Is_periodic[2]=0;
mpi_Reorder = 0;
if(mpi_Rank == 0)
{
std::cout<<"Total MPI processes = "<<mpi_Size<<std::endl;
std::cout<<"Thread level provided = "<<mpi_Provided_level<<std::endl;
}
// if(mpi_Rank == 0)
// {
// std::cout<<"Total MPI processes = "<<mpi_Size<<std::endl;
// std::cout<<"Thread level provided = "<<mpi_Provided_level<<std::endl;
// }
/*------------------------------------------------------------*/
/* Create MPI topology, find coords */
/*------------------------------------------------------------*/
/* Create MPI topology, find coords */
/*------------------------------------------------------------*/
MPI_Cart_create(mpi_Comm, 3, mpi_Dims, mpi_Is_periodic, mpi_Reorder, &mpi_Cart_comm);
......@@ -477,7 +483,7 @@ int main( int argc, char* argv[] )
/* OpenMP threads set */
/*--------------------------------------------*/
omp_set_num_threads(omp_num_threads);
//omp_set_num_threads(omp_num_threads);
/*--------------------------------------------*/
/* Create Microenvironment */
......@@ -492,15 +498,19 @@ int main( int argc, char* argv[] )
microenvironment.mesh.units = "microns";
microenvironment.time_units = "minutes";
if(mpi_Rank == 0)
std::cout << "Total Voxels = " << microenvironment.mesh.voxels.size() << std::endl;
/*if(mpi_Rank == 0)
std::cout << "Total Voxels = " << microenvironment.mesh.voxels.size() << std::endl;*/
/*------------------------------------------------------------*/
/* Resize local Microenvironment on each process */
/*------------------------------------------------------------*/
t_start = MPI_Wtime();
microenvironment.resize_space_uniform( minX,maxX,minY,maxY,minZ,maxZ, mesh_resolution, mpi_Dims, mpi_Coords);
microenvironment.display_information( std::cout );
t_end = MPI_Wtime();
if(mpi_Rank==0)
std::cout<<"TIME FOR RESIZING MICROENVIRONMENT = "<< (t_end-t_start)<< std::endl;
//microenvironment.display_information( std::cout );
std::vector<double> center(3);
......@@ -508,20 +518,30 @@ int main( int argc, char* argv[] )
center[1] = (microenvironment.mesh.bounding_box[1]+microenvironment.mesh.bounding_box[4])/2;
center[2] = (microenvironment.mesh.bounding_box[2]+microenvironment.mesh.bounding_box[5])/2;
double stddev_squared = -100.0 * 100.0;
std::vector<double> one( microenvironment.density_vector(0).size() , 1.0 );
int num_voxels = microenvironment.number_of_voxels();
std::cout << "Total voxels are = "<<num_voxels;
//#pragma omp parallel for
std::vector<double> one( microenvironment.density_vector(0).size() , 1.0 );
// int num_voxels = microenvironment.number_of_voxels();
// std::cout << "Total voxels are = "<<num_voxels;
t_start = MPI_Wtime();
#pragma omp parallel for
for( int i=0; i < microenvironment.number_of_voxels() ; i++ )
{
std::vector<double> displacement = microenvironment.voxels(i).center - center;
std::vector<double> displacement = microenvironment.voxels(i).center - center;
double distance_squared = norm_squared( displacement );
double coeff = distance_squared;
coeff /= stddev_squared;
microenvironment.density_vector(i)[0]= exp( coeff );
}
microenvironment.write_to_matlab( "initial_concentration.mat", mpi_Rank, mpi_Size, mpi_Cart_comm );
t_end = MPI_Wtime();
if(mpi_Rank==0)
std::cout<<"TIME FOR GENERATING GAUSSIAN PROFILE = "<< (t_end-t_start)<< std::endl;
t_start = MPI_Wtime();
microenvironment.write_to_matlab( "initial_concentration.mat", mpi_Rank, mpi_Size, mpi_Cart_comm );
t_end = MPI_Wtime();
if(mpi_Rank==0)
std::cout<<"TIME FOR WRITING INITIAL CONCENTRATION FILE = "<< (t_end-t_start)<< std::endl;
// register the diffusion solver
microenvironment.diffusion_decay_solver = diffusion_decay_solver__constant_coefficients_LOD_3D;
......@@ -531,53 +551,48 @@ int main( int argc, char* argv[] )
microenvironment.decay_rates[0] = 0.01;
// display information
microenvironment.display_information( std::cout );
//microenvironment.display_information( std::cout );
double dt = 0.01;
double cell_radius = 5;
int num_sources = 500;
int num_sinks = 500;
t_start = MPI_Wtime();
create_point_sources(cell_radius, dt, num_sources, microenvironment, mpi_Dims, mpi_Rank, mpi_Size, mpi_Cart_comm);
create_point_sinks(cell_radius, dt, num_sinks, microenvironment, mpi_Dims, mpi_Rank, mpi_Size, mpi_Cart_comm);
/*for(int i=0; i< 500;i++)
{
std::vector<double> tempPoint(3,0.0);
for( int j=0; j < 3 ; j++ )
{ tempPoint[j] = UniformRandom()*1000; }
Basic_Agent * temp_point_source = create_basic_agent();
temp_point_source->register_microenvironment(&microenvironment);
temp_point_source->assign_position(tempPoint);
temp_point_source->set_total_volume( (4.0/3.0)*pi*pow(cell_radius,3.0) );
(*temp_point_source->secretion_rates)[0]=10;
(*temp_point_source->saturation_densities)[0]=1;
temp_point_source->set_internal_uptake_constants(dt);
for( int j=0; j < 3 ; j++ )
{ tempPoint[j] = UniformRandom()*1000; }
Basic_Agent * temp_point_sink = create_basic_agent();
temp_point_sink->register_microenvironment(&microenvironment);
temp_point_sink->assign_position(tempPoint);
temp_point_sink->set_total_volume( (4.0/3.0)*pi*pow(cell_radius,3.0) );
(*temp_point_sink->uptake_rates)[0]=0.8;
temp_point_sink->set_internal_uptake_constants(dt);
}
*/
t_end = MPI_Wtime();
if(mpi_Rank==0)
std::cout<<"TIME FOR CREATING ALL BASIC AGENTS = "<< (t_end-t_start)<< std::endl;
double t = 0.0;
double t_max=5;
while( t < t_max )
t_start = MPI_Wtime();
while( t < t_max )
{
microenvironment.simulate_cell_sources_and_sinks( dt );
microenvironment.simulate_diffusion_decay( dt, mpi_Size, mpi_Rank, mpi_Coords, mpi_Dims, mpi_Cart_comm );
t += dt;
}
microenvironment.write_to_matlab( "final.mat", mpi_Rank, mpi_Size, mpi_Cart_comm ); //Remember to use Parallel Version !
std::cout<<"done!"<<std::endl;
t_end = MPI_Wtime();
if(mpi_Rank==0)
std::cout<<"TIME FOR SIMULATING (SOURCES+SINKS+DIFFUSION) = "<< (t_end-t_start)<< std::endl;
t_start = MPI_Wtime();
microenvironment.write_to_matlab( "final.mat", mpi_Rank, mpi_Size, mpi_Cart_comm ); //Remember to use Parallel Version !
t_end = MPI_Wtime();
if(mpi_Rank==0)
std::cout<<"TIME FOR WRITING FINAL FILE = "<< (t_end-t_start)<< std::endl;
MPI_Barrier(mpi_Comm);
t_total_end = MPI_Wtime();
if(mpi_Rank == 0)
std::cout<<"TOTAL PROGRAM EXECUTION TIME = " <<(t_total_end-t_total_start)<<std::endl;
//std::cout<<"done!"<<std::endl;
/*-------------------------------------------------------------------------*/
......
final.mat
View file @ c8a5bf8b
No preview for this file type
output-7584197 0 → 100644
View file @ c8a5bf8b
TIME FOR RESIZING MICROENVIRONMENT = 0.758827
TIME FOR GENERATING GAUSSIAN PROFILE = 0.00794553
TIME FOR WRITING INITIAL CONCENTRATION FILE = 0.34451
TIME FOR CREATING ALL BASIC AGENTS = 0.000949931
TIME FOR SIMULATING (SOURCES+SINKS+DIFFUSION) = 8.48919
TIME FOR WRITING FINAL FILE = 0.280464
TOTAL PROGRAM EXECUTION TIME = 9.88661
script_paraprof.sh 0 → 100644
View file @ c8a5bf8b
#!/bin/bash
#SBATCH -n 48
#SBATCH -t 00:15:00
#SBATCH --cpus-per-task=1
#SBATCH -o output-%j
#SBATCH -e error-%j
#SBATCH --x11=batch
# set application and parameters
paraprof
#jumpshot tau.slog2
script_tutorial1.sh
View file @ c8a5bf8b
#!/bin/bash
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=8
#SBATCH --cpus-per-task=1
#SBATCH -t 00:05:00
#SBATCH --nodes=4
#SBATCH --ntasks-per-node=2
#SBATCH --cpus-per-task=24
#SBATCH -t 00:15:00
#SBATCH -o output-%j
#SBATCH -e error-%j
#SBATCH --exclusive
mpiexec ./examples/tutorial1
#export OMP_SCHEDULE=STATIC
export OMP_DISPLAY_ENV=true
export OMP_NUM_THREADS=24
export OMP_PROC_BIND=spread
#export OMP_PLACES="{0:1}:48:1"
export OMP_PLACES=threads
mpiexec --map-by ppr:1:socket:pe=24 --report-bindings ./examples/tutorial1
#mpiexec --map-by socket --bind-to core --report-bindings ./examples/tutorial1
#mpiexec --map-by node --bind-to none --report-bindings ./examples/tutorial1
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment