pic_time_step.cpp 18.9 KB
Newer Older
1
2
//the file contains parts of PIC::TimeStep

3
4
5
#include <dirent.h>
#include <errno.h>

6
#include "pic.h"
7
8
#include "global.h"
#include "PhotolyticReactions.h"
9
10
11

void PIC::TimeStepInternal::PrintTimeStep() {
  using namespace PIC;
12

13
14
15
16
17
  if (ThisThread==0) {
    static int InteractionCouinter=0;
    time_t TimeValue=time(NULL);
    tm *ct=localtime(&TimeValue);

18
    if ((SamplingMode!=_DISABLED_SAMPLING_MODE_)&&(SamplingMode!=_TEMP_DISABLED_SAMPLING_MODE_)) {
19
      printf("$PREFIX: (%i/%i %i:%i:%i), Iteration: %i  (current sample length:%ld, %ld iterations to the next output)\n",ct->tm_mon+1,ct->tm_mday,ct->tm_hour,ct->tm_min,ct->tm_sec,InteractionCouinter,RequiredSampleLength,RequiredSampleLength-CollectingSampleCounter);
20
21
22
23
24
    }
    else {
      printf("$PREFIX: (%i/%i %i:%i:%i), Iteration: %i  (sampling is disabled)\n",ct->tm_mon+1,ct->tm_mday,ct->tm_hour,ct->tm_min,ct->tm_sec,InteractionCouinter);
    }

25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
    InteractionCouinter++;
  }
}


//===============================================================================================
//recover the sampling data from the sampling data restart file, print the TECPLOT files and quit
void PIC::TimeStepInternal::RecoverSamplingDataRestart() {
  static bool RestartFileReadFlag=false;

  if (RestartFileReadFlag==false) {
    RestartFileReadFlag=true;

    //prepare the list of the files/speces which data fille be revobered and saved
    list<pair<string,list<int> > > SampledDataRecoveryTable;

    if (Restart::SamplingData::DataRecoveryManager==NULL) {
      //no user-defined fucntion to create the 'SampledDataRecoveringTable' is provided
      pair<string,list<int> > Table;

      for (int s=0;s<PIC::nTotalSpecies;s++) Table.second.push_back(s);

      Table.first=Restart::SamplingData::RestartFileName;
      SampledDataRecoveryTable.push_back(Table);
    }
    else Restart::SamplingData::DataRecoveryManager(SampledDataRecoveryTable,Restart::SamplingData::minReadFileNumber,Restart::SamplingData::maxReadFileNumber);

    //iterate through SampledDataRecoveryTable
    list<pair<string,list<int> > >::iterator RecoveryEntry;

    for (RecoveryEntry=SampledDataRecoveryTable.begin();RecoveryEntry!=SampledDataRecoveryTable.end();RecoveryEntry++) {
      Restart::SamplingData::Read(RecoveryEntry->first.c_str());
      MPI_Barrier(MPI_GLOBAL_COMMUNICATOR);

      for (list<int>::iterator s=RecoveryEntry->second.begin();s!=RecoveryEntry->second.end();s++) {
        char fname[_MAX_STRING_LENGTH_PIC_],ChemSymbol[_MAX_STRING_LENGTH_PIC_];

        PIC::MolecularData::GetChemSymbol(ChemSymbol,*s);
        sprintf(fname,"RECOVERED.%s.%s.s=%i.dat",RecoveryEntry->first.c_str(),ChemSymbol,*s);
        PIC::Mesh::mesh->outputMeshDataTECPLOT(fname,*s);

        //preplot the recovered file if needed
        if (Restart::SamplingData::PreplotRecoveredData==true) {
          char cmd[_MAX_STRING_LENGTH_PIC_];

          sprintf(cmd,"preplot %s",fname);
          system(cmd);
        }
      }
    }

    MPI_Barrier(MPI_GLOBAL_COMMUNICATOR);

    if (_PIC_RECOVER_SAMPLING_DATA_RESTART_FILE__EXECUTION_MODE_ == _PIC_RECOVER_SAMPLING_DATA_RESTART_FILE__EXECUTION_MODE__STOP_) {
      if (PIC::ThisThread==0) {
        printf("The sampled data from restart file/files\n");
        for (RecoveryEntry=SampledDataRecoveryTable.begin();RecoveryEntry!=SampledDataRecoveryTable.end();RecoveryEntry++) printf("%s\n",RecoveryEntry->first.c_str());
        printf("is successfully recoved. Execution is complete. See you later :-)\n");
      }

      MPI_Finalize();
      exit(EXIT_SUCCESS);
    }
  }
}


//===============================================================================================
//recover the particle data restart file
void PIC::TimeStepInternal::ReadParticleDataRestartFile() {
  static bool RestartFileReadFlag=false;

  using namespace PIC;

  if (RestartFileReadFlag==false) {
    RestartFileReadFlag=true;

    Restart::ReadParticleData(Restart::recoverParticleDataRestartFileName);
    MPI_Barrier(MPI_GLOBAL_COMMUNICATOR);
  }
}

//===============================================================================================
//save the particle data restart file
void PIC::TimeStepInternal::SaveParticleRestartFile() {
110
  static long int IterationCounter=0;
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
  static int saveRestartFileCounter=0;
  
  using namespace PIC;

  IterationCounter++;

  if (IterationCounter%Restart::ParticleRestartAutosaveIterationInterval==0) {
    char fname[_MAX_STRING_LENGTH_PIC_];

    if (Restart::ParticleDataRestartFileOverwriteMode==true) sprintf(fname,"%s",Restart::saveParticleDataRestartFileName);
    else sprintf(fname,"%s.restart=%i",Restart::saveParticleDataRestartFileName,saveRestartFileCounter);

    Restart::SaveParticleData(fname);
    saveRestartFileCounter++;

    MPI_Barrier(MPI_GLOBAL_COMMUNICATOR);
  }
}

//===============================================================================================
131
//simulate particle collisions
132
133
134
void PIC::TimeStepInternal::ParticleCollisions(double &ParticleCollisionTime) {
  SetExitErrorCode(__LINE__,_PIC__EXIT_CODE__LAST_FUNCTION__PIC_TimeStep_);
  ParticleCollisionTime=MPI_Wtime();
135
136
137

  if (_PIC_LOGGER_MODE_==_PIC_MODE_ON_) {
    PIC::Debugger::LoggerData.erase();
138
    PIC::Debugger::logger.func_enter(__LINE__,"PIC::TimeStep()",&PIC::Debugger::LoggerData,0,_PIC_LOGGER_TIME_LIMIT_);
139
  }
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
  
  switch (_PIC__PARTICLE_COLLISION_MODEL_) {
  case _PIC__PARTICLE_COLLISION_MODEL__NTC_:
    PIC::MolecularCollisions::ParticleCollisionModel::ntc();
    break;
    
  case _PIC__PARTICLE_COLLISION_MODEL__MF_:
    PIC::MolecularCollisions::ParticleCollisionModel::mf();
    break;
    
  case _PIC__PARTICLE_COLLISION_MODEL__USER_DEFINED_:
    exit(__LINE__,__FILE__,"Error: the option is not implemented");
    break;
    
  default:  
    exit(__LINE__,__FILE__,"Error: the option is not implemented");
  }
157
158
159
160
161

  if (_PIC_LOGGER_MODE_==_PIC_MODE_ON_) {
    PIC::Debugger::logger.func_exit();
  }

162
  ParticleCollisionTime=MPI_Wtime()-ParticleCollisionTime;
163
}
164
165
166



167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
//===============================================================================================
//injection boundary conditions
void PIC::TimeStepInternal::ParticleInjectionBC(double &InjectionBoundaryTime) {
  InjectionBoundaryTime=MPI_Wtime();

  //inject particle through the domain's boundaries
  SetExitErrorCode(__LINE__,_PIC__EXIT_CODE__LAST_FUNCTION__PIC_TimeStep_);

  PIC::BC::InjectionBoundaryConditions();

  //inject particles into the volume of the domain
  #if _PIC_VOLUME_PARTICLE_INJECTION_MODE_ == _PIC_VOLUME_PARTICLE_INJECTION_MODE__ON_
  if (PIC::VolumeParticleInjection::nRegistratedInjectionProcesses!=0) PIC::BC::nTotalInjectedParticles+=PIC::VolumeParticleInjection::InjectParticle();
  #endif

  //call a user-defined injection function
  if (PIC::BC::UserDefinedParticleInjectionFunction!=NULL) {
    SetExitErrorCode(__LINE__,_PIC__EXIT_CODE__LAST_FUNCTION__PIC_TimeStep_);
    PIC::BC::nTotalInjectedParticles+=PIC::BC::UserDefinedParticleInjectionFunction();
  }

  //the extra injection process by the exosphere model (src/models/exosphere)
  if (BC::ExosphereModelExtraInjectionFunction!=NULL) {
    SetExitErrorCode(__LINE__,_PIC__EXIT_CODE__LAST_FUNCTION__PIC_TimeStep_);
    PIC::BC::nTotalInjectedParticles+=BC::ExosphereModelExtraInjectionFunction();
  }

  InjectionBoundaryTime=MPI_Wtime()-InjectionBoundaryTime;
}

197

198
199
200
201
202
203
204
205
206
207
208
//===============================================================================================
//initialization at the first call of PIC::TimeStep()
void PIC::TimeStepInternal::Init() {
  if (_SIMULATION_TIME_STEP_MODE_ != _SINGLE_GLOBAL_TIME_STEP_) exit(__LINE__,__FILE__,"Error: the option is only implemented for single global time step");

  if (PIC::Sampling::SampleTimeInterval<0) exit(__LINE__,__FILE__,"Error: the sample time interval is negative");

  if (PIC::ParticleWeightTimeStep::GetGlobalTimeStep(0)<0) {
    exit(__LINE__,__FILE__,"Error: the global time step is negative");
  }
  else {
209
    PIC::RequiredSampleLength=(long int)(PIC::Sampling::SampleTimeInterval/PIC::ParticleWeightTimeStep::GetGlobalTimeStep(0)+0.5);
210

211
212
213
214
215
    if (PIC::RequiredSampleLength==0) {
      char msg[600];
      sprintf(msg,"Error: the required sample length is 0, PIC::Sampling::SampleTimeInterval:%e, PIC::ParticleWeightTimeStep::GetGlobalTimeStep(0):%e", PIC::Sampling::SampleTimeInterval,PIC::ParticleWeightTimeStep::GetGlobalTimeStep(0));
      exit(__LINE__,__FILE__,msg);
    }
216

217
    if (PIC::ThisThread==0) printf("PIC::RequiredSampleLength is set to %ld \n", PIC::RequiredSampleLength);
218
219
  } 
}
220

221
222
223
//===============================================================================================
//sampling of the particle properties and creating an output file
void PIC::TimeStepInternal::Sampling(double &SamplingTime) {
224
225
226

  using namespace PIC;

227
  if ((SamplingMode==_DISABLED_SAMPLING_MODE_)||(SamplingMode==_TEMP_DISABLED_SAMPLING_MODE_)) {
228
    SamplingTime=0.0;
229

230
231
232
    return;
  }

233
234
  SetExitErrorCode(__LINE__,_PIC__EXIT_CODE__LAST_FUNCTION__PIC_TimeStep_);
  SamplingTime=MPI_Wtime();
235

236
237
238
239
240
241
242
243
244
245
246
247
  if ((_PIC_DEBUGGER_MODE_ == _PIC_DEBUGGER_MODE_ON_) && 
      (_PIC_DEBUGGER_MODE__SAMPLING_BUFFER_VALUE_RANGE_CHECK_ == _PIC_DEBUGGER_MODE__VARIABLE_VALUE_RANGE_CHECK_ON_)) {
    Sampling::CatchOutLimitSampledValue();
  }

  timing_start("PT::Sampling");
  PIC::Sampling::Sampling();
  timing_stop("PT::Sampling");


  SamplingTime=MPI_Wtime()-SamplingTime;
}
248
249


250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
//===============================================================================================
//Background atmosphere model
void PIC::TimeStepInternal::BackgroundAtmosphereModel(double& BackgroundAtmosphereCollisionTime) {
  SetExitErrorCode(__LINE__,_PIC__EXIT_CODE__LAST_FUNCTION__PIC_TimeStep_);
  BackgroundAtmosphereCollisionTime=MPI_Wtime();

  using namespace PIC;
 
  #if _PIC_BACKGROUND_ATMOSPHERE_MODE_ == _PIC_BACKGROUND_ATMOSPHERE_MODE__ON_
  #if _PIC_BACKGROUND_ATMOSPHERE__COLLISION_MODEL_ == _PIC_BACKGROUND_ATMOSPHERE__COLLISION_MODEL__PARTICLE_COLLISIONS_
  MolecularCollisions::BackgroundAtmosphere::CollisionProcessor();
  #elif _PIC_BACKGROUND_ATMOSPHERE__COLLISION_MODEL_ == _PIC_BACKGROUND_ATMOSPHERE__COLLISION_MODEL__STOPPING_POWER_
  MolecularCollisions::BackgroundAtmosphere::StoppingPowerProcessor();
  #else
  exit(__LINE__,__FILE__,"Error: the option is unknown");
  #endif //_PIC_BACKGROUND_ATMOSPHERE__COLLISION_MODEL_
  #endif

  BackgroundAtmosphereCollisionTime=MPI_Wtime()-BackgroundAtmosphereCollisionTime;
}


//===============================================================================================
//Simulate photolytic reactions 
void PIC::TimeStepInternal::PhtolyticReactions(double &PhotoChemistryTime) {
  SetExitErrorCode(__LINE__,_PIC__EXIT_CODE__LAST_FUNCTION__PIC_TimeStep_);
  PhotoChemistryTime=MPI_Wtime();

  ChemicalReactions::PhotolyticReactions::ExecutePhotochemicalModel();

  PhotoChemistryTime=MPI_Wtime()-PhotoChemistryTime;
  RunTimeSystemState::CumulativeTiming::PhotoChemistryTime+=PhotoChemistryTime;
}

//===============================================================================================
//perform user-define processing of the model particles
void PIC::TimeStepInternal::UserDefinedParticleProcessing(double& UserDefinedParticleProcessingTime) {
  SetExitErrorCode(__LINE__,_PIC__EXIT_CODE__LAST_FUNCTION__PIC_TimeStep_);
  UserDefinedParticleProcessingTime=MPI_Wtime();

  PIC::UserParticleProcessing::Processing();
291

292
293
  UserDefinedParticleProcessingTime=MPI_Wtime()-UserDefinedParticleProcessingTime;
}
294

295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
//===============================================================================================
//check particle lists
void PIC::TimeStepInternal::CheckParticleLists() {
  SetExitErrorCode(__LINE__,_PIC__EXIT_CODE__LAST_FUNCTION__PIC_TimeStep_);

  switch (_PIC_PARTICLE_LIST_ATTACHING_) {
  case _PIC_PARTICLE_LIST_ATTACHING_FL_SEGMENT_:
    PIC::FieldLine::CheckParticleList(); 
    break;

  case _PIC_PARTICLE_LIST_ATTACHING_NODE_: 
    PIC::ParticleBuffer::CheckParticleList();
    break;
  default:
    exit(__LINE__,__FILE__,"Error: the option is unknown");
  }
}

//===============================================================================================
//execution track specific to the default model setting
void PIC::TimeStepInternal::ExecutionTrackDefault(double& ParticleMovingTime,double& ParticleExchangeTime) {
316
317
  if (_PIC_LOGGER_MODE_==_PIC_MODE_ON_) {
    Debugger::LoggerData.erase();
318
    Debugger::logger.func_enter(__LINE__,"PIC::TimeStepInternal::ExecutionTrackDefault()",&Debugger::LoggerData,0,_PIC_LOGGER_TIME_LIMIT_);
319
320
321
322
323
324
325
326
  }

  if (_PIC_LOGGER_MODE_==_PIC_MODE_ON_) {
    PIC::Debugger::LoggerData.erase();
    sprintf(PIC::Debugger::LoggerData.msg,"PIC::TimeStepInternal::ExecutionTrackDefault)): call particle mover");
    PIC::Debugger::logger.add_data_point(__LINE__,&PIC::Debugger::LoggerData);
  }

327
328
329
  ParticleMovingTime=MPI_Wtime();
  PIC::Mover::MoveParticles();

330
331
332
333
334
335
  if (_PIC_LOGGER_MODE_==_PIC_MODE_ON_) {
    PIC::Debugger::LoggerData.erase();
    sprintf(PIC::Debugger::LoggerData.msg,"PIC::TimeStepInternal::ExecutionTrackDefault)): call exange particle procedure");
    PIC::Debugger::logger.add_data_point(__LINE__,&PIC::Debugger::LoggerData);
  }

336
337
338
339
340
341
  if (_PIC_BC__PERIODIC_MODE_==_PIC_BC__PERIODIC_MODE_ON_) {
    PIC::BC::ExternalBoundary::Periodic::ExchangeParticles();
  }

  ParticleMovingTime=MPI_Wtime()-ParticleMovingTime;

342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
  //increase the number of particles if needed
  switch (PIC::ParticleSplitting::Mode) {
  case PIC::ParticleSplitting::_disactivated:
    //do nothing
    break;
  case PIC::ParticleSplitting::_VelocityShift:
    PIC::ParticleSplitting::Split::SplitWithVelocityShift(PIC::ParticleSplitting::particle_num_limit_min,PIC::ParticleSplitting::particle_num_limit_max);
    break;
  case PIC::ParticleSplitting::_Scatter:
    PIC::ParticleSplitting::Split::Scatter(PIC::ParticleSplitting::particle_num_limit_min,PIC::ParticleSplitting::particle_num_limit_max);
    break;
  default:
    exit(__LINE__,__FILE__,"Error: the option is unlnown");
  }


358
  //check the consistence of the particles lists
359
  if ((_PIC_DEBUGGER_MODE_ == _PIC_DEBUGGER_MODE_ON_)&&(_CUDA_MODE_ == _OFF_)) {
360
361
362
363
364
365
366
367
    CheckParticleLists(); 
  }

  //syncronize processes and exchange particle data
  SetExitErrorCode(__LINE__,_PIC__EXIT_CODE__LAST_FUNCTION__PIC_TimeStep_);
  ParticleExchangeTime=MPI_Wtime();
  PIC::Parallel::ExchangeParticleData();
  ParticleExchangeTime=MPI_Wtime()-ParticleExchangeTime;
368
369
370
371

  if (_PIC_LOGGER_MODE_==_PIC_MODE_ON_) {
    Debugger::logger.func_exit();
  }
372
373
374
375
376
377
}

  

//===============================================================================================
//execution track specific to the ECSIM field solver
Valeriy M Tenishev's avatar
Valeriy M Tenishev committed
378
void PIC::TimeStepInternal::ExecutionTrackFieldSolverECSIM(double& ParticleMovingTime,double& ParticleExchangeTime,double& FieldSolverTime) {
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441

  //if the periodeic boundary conditions are in use -> exchange particles between 'real' and 'ghost' blocks

  using namespace PIC;

  ParticleMovingTime=0.0,ParticleExchangeTime=0.0,FieldSolverTime=0.0;

#if _PIC_FIELD_SOLVER_MODE_ != _PIC_FIELD_SOLVER_MODE__OFF_

  auto FIELD_SOLVER_Task3 = [=] {
    PIC::BC::ExternalBoundary::Periodic::ExchangeParticles();
    PIC::Parallel::UpdateGhostBlockData();
  };

  //#if _CUDA_MODE_ == _OFF_ 
  FIELD_SOLVER_Task3();
  // #else 
  //exit(__LINE__,__FILE__,"Error: not implemented");
  //#endif


  //update elecrtic and magnetic fields
  // #if _PIC_FIELD_SOLVER_MODE_==_PIC_FIELD_SOLVER_MODE__OFF_
  //do nothing
  // #else
  SetExitErrorCode(__LINE__,_PIC__EXIT_CODE__LAST_FUNCTION__PIC_TimeStep_);
  FieldSolverTime=MPI_Wtime();

  auto FIELD_SOLVER_Task4 = [=] () {
    switch (_PIC_FIELD_SOLVER_MODE_) {
    case _PIC_FIELD_SOLVER_MODE__ELECTROMAGNETIC__ECSIM_:
      FieldSolver::Electromagnetic::ECSIM::TimeStep();
      break;
    default:
      exit(__LINE__,__FILE__,"Error: unknown value of _PIC_FIELD_SOLVER_MODE_");
    }
  };

  //#if _CUDA_MODE_ == _OFF_ 
  FIELD_SOLVER_Task4();
  //#else 
  //exit(__LINE__,__FILE__,"Error: not implemented");
  //#endif


  //#if _PIC_FIELD_SOLVER_MODE_==_PIC_FIELD_SOLVER_MODE__ELECTROMAGNETIC__ECSIM_
  ParticleMovingTime=MPI_Wtime();
  PIC::FieldSolver::Electromagnetic::ECSIM::CumulativeTiming::ParticleMoverTime.Start();

  auto FIELD_SOLVER_Task5 = [=] () {
    PIC::Mover::MoveParticles();
  };

  //#if _CUDA_MODE_ == _OFF_ 
  FIELD_SOLVER_Task5();
  //#else 
  //exit(__LINE__,__FILE__,"Error: not implemented");
  //#endif

  ParticleMovingTime=MPI_Wtime()-ParticleMovingTime;
  RunTimeSystemState::CumulativeTiming::ParticleMovingTime+=ParticleMovingTime;

  //check the consistence of the particles lists
442
  if ((_PIC_DEBUGGER_MODE_ == _PIC_DEBUGGER_MODE_ON_)&&(_CUDA_MODE_ == _OFF_)) {
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
    PIC::ParticleBuffer::CheckParticleList();
  }

  //syncronize processors and exchange particle data
  SetExitErrorCode(__LINE__,_PIC__EXIT_CODE__LAST_FUNCTION__PIC_TimeStep_);
  ParticleExchangeTime=MPI_Wtime();


  //#if _CUDA_MODE_ == _OFF_ 
  PIC::Parallel::ExchangeParticleData();
  //#else 
  //exit(__LINE__,__FILE__,"Error: not implemented");
  //#endif

  PIC::FieldSolver::Electromagnetic::ECSIM::CumulativeTiming::ParticleMoverTime.UpdateTimer();
  ParticleExchangeTime=MPI_Wtime()-ParticleExchangeTime;
  RunTimeSystemState::CumulativeTiming::ParticleExchangeTime+=ParticleExchangeTime;

  auto FIELD_SOLVER_Task6 = [=] () {
    if (_PIC_BC__PERIODIC_MODE_==_PIC_BC__PERIODIC_MODE_OFF_ )
      PIC::FieldSolver::Electromagnetic::ECSIM::setParticle_BC();

    if (PIC::FieldSolver::Electromagnetic::ECSIM::DoDivECorrection)
      PIC::FieldSolver::Electromagnetic::ECSIM::divECorrection();
  };

  //#if _CUDA_MODE_ == _OFF_ 
  FIELD_SOLVER_Task6();
  //#else 
  //exit(__LINE__,__FILE__,"Error: not implemented");
  //#endif


  if (_PIC_BC__PERIODIC_MODE_==_PIC_BC__PERIODIC_MODE_ON_ ) { 
    auto FIELD_SOLVER_Task7 = [=] () {
      PIC::BC::ExternalBoundary::Periodic::ExchangeParticles();
    };

    //#if _CUDA_MODE_ == _OFF_ 
    FIELD_SOLVER_Task7();
    //#else
    //exit(__LINE__,__FILE__,"Error: not implemented");
    //#endif
  }


  auto FIELD_SOLVER_Task8 = [=] () {
    PIC::FieldSolver::Electromagnetic::ECSIM::UpdateJMassMatrix();
  };

  //#if _CUDA_MODE_ == _OFF_
  FIELD_SOLVER_Task8();
  //#else 
  //exit(__LINE__,__FILE__,"Error: not implemented");
  //#endif


  PIC::CPLR::FLUID::iCycle++;
  {// Output

    //#if _CUDA_MODE_ == _ON_
    //exit(__LINE__,__FILE__,"Error: not implemented");
    //#endif

    double timeNow = PIC::ParticleWeightTimeStep::GlobalTimeStep[0]*PIC::CPLR::FLUID::iCycle;    
    if (PIC::ThisThread==0) printf("pic.cpp timeNow:%e,iCycle:%d\n",timeNow,PIC::CPLR::FLUID::iCycle);
    PIC::CPLR::FLUID::write_output(timeNow);
    PIC::CPLR::FLUID::check_max_mem_usage();
    PIC::FieldSolver::Electromagnetic::ECSIM::CumulativeTiming::Print();

  }

  // #endif
  //if the periodeic boundary conditions are in use -> exchange new values of the electric and magnetic fields between 'real' and 'ghost' blocks/
  //  #if _PIC_FIELD_SOLVER_MODE_ == _PIC_FIELD_SOLVER_MODE__ELECTROMAGNETIC__ECSIM_

  //#if _CUDA_MODE_ == _ON_
  //exit(__LINE__,__FILE__,"Error: not implemented");
  //#endif

  PIC::Parallel::ProcessBlockBoundaryNodes(); 

  //  #endif

  FieldSolverTime=MPI_Wtime()-FieldSolverTime;
  RunTimeSystemState::CumulativeTiming::FieldSolverTime+=FieldSolverTime;
#endif //_PIC_FIELD_SOLVER_MODE_
}