Fixing refinement issue on DisplacedMesh.

The issue is that mesh refinement is always followed by partitioning,
which requires recomputation of the Hilbert indices.  Since the
DisplacedSystem mesh is displaced inconsistently on different
processors, this can lead to inconsistent Hilbert keys being computed
on different processors, and ultimately an error during partitioning.

I tried several more or less complicated fixes for this, but the
simplest approach seems to be to simply 'undisplace' the
DisplacedSystem mesh prior to performing adaptive refinement, and then
let it be naturally 're-displaced' when the DisplacedSystem is
reinitialized.  This seems to work fairly well, and fixes the
PF_displaced_mesh_test in MARMOT, which was not working in parallel.
We can also undisplace the mesh using threads to help speed things up.

Refs #4404

framework/include/base/DisplacedProblem.h

@@ -174,6 +174,12 @@ class DisplacedProblem : public SubProblem
    */
   virtual void registerRestartableData(std::string name, RestartableDataValue * data, THREAD_ID tid);
 
+  /**
+   * Resets the displaced mesh to the reference mesh.  Required when
+   * refining the DisplacedMesh.
+   */
+  void undisplaceMesh();
+
 protected:
   FEProblem & _mproblem;
   MooseMesh & _mesh;

framework/include/base/UpdateDisplacedMeshThread.h

@@ -27,6 +27,7 @@ class UpdateDisplacedMeshThread
   UpdateDisplacedMeshThread(DisplacedProblem & problem);
 
   void operator() (const SemiLocalNodeRange & range) const;
+  void operator() (const NodeRange & range) const;
 
 protected:
   DisplacedProblem & _problem;

framework/src/base/Adaptivity.C

@@ -92,8 +92,14 @@ Adaptivity::init(unsigned int steps, unsigned int initial_steps)
 
     if (!_displaced_mesh_refinement)
       _displaced_mesh_refinement = new MeshRefinement(_displaced_problem->mesh());
+
+    // The periodic boundaries pointer allows the MeshRefinement
+    // object to determine elements which are "topological" neighbors,
+    // i.e. neighbors across periodic boundaries, for the purposes of
+    // refinement.
     _displaced_mesh_refinement->set_periodic_boundaries_ptr(_subproblem.getNonlinearSystem().dofMap().get_periodic_boundaries());
 
+    // TODO: This is currently an empty function on the DisplacedProblem... could it be removed?
     _displaced_problem->initAdaptivity();
   }
 }
@@ -154,15 +160,26 @@ Adaptivity::adaptMesh()
         _displaced_mesh_refinement->flag_elements_by_error_fraction (*_error);
     }
 
+    // If the DisplacedProblem is active, undisplace the DisplacedMesh
+    // in preparation for refinement.  We can't safely refine the
+    // DisplacedMesh directly, since the Hilbert keys computed on the
+    // inconsistenly-displaced Mesh are different on different
+    // processors, leading to inconsistent Hilbert keys.  We must do
+    // this before the undisplaced Mesh is refined, so that the
+    // element and node numbering is still consistent.
+    if (_displaced_problem)
+      _displaced_problem->undisplaceMesh();
+
     // Perform refinement and coarsening
     meshChanged = _mesh_refinement->refine_and_coarsen_elements();
 
-    if (_displaced_problem)
+    if (_displaced_problem && meshChanged)
     {
+      // Now do refinement/coarsening
       bool dispMeshChanged = _displaced_mesh_refinement->refine_and_coarsen_elements();
-      if ((meshChanged && !dispMeshChanged) ||
-          (!meshChanged && dispMeshChanged))
-        mooseError("If either undisplaced mesh or displaced mesh changes due to adaptivity, both must change");
+
+      // Since the undisplaced mesh changed, the displaced mesh better have changed!
+      mooseAssert(dispMeshChanged, "Undisplaced mesh changed, but displaced mesh did not!");
     }
 
     if (meshChanged && _print_mesh_changed)
@@ -200,7 +217,12 @@ Adaptivity::uniformRefine(unsigned int level)
   // we have to go step by step so EquationSystems::reinit() won't freak out
   for (unsigned int i = 0; i < level; i++)
   {
+    // See comment above about why refining the displaced mesh is potentially unsafe.
+    if (_displaced_problem)
+      _displaced_problem->undisplaceMesh();
+
     mesh_refinement.uniformly_refine(1);
+
     if (_displaced_problem)
       displaced_mesh_refinement.uniformly_refine(1);
     _subproblem.meshChanged();

framework/src/base/DisplacedProblem.C

@@ -575,3 +575,26 @@ DisplacedProblem::registerRecoverableData(std::string name)
 
   _mproblem.registerRecoverableData(name);
 }
+
+void
+DisplacedProblem::undisplaceMesh()
+{
+  // If undisplaceMesh() is called during initial adaptivity, it is
+  // not valid to call _mesh.getActiveSemiLocalNodeRange() since it is
+  // not set up yet.  So we are creating the Range by hand.
+  //
+  // We must undisplace *all* our nodes to the _ref_mesh
+  // configuration, not just the local ones, since the partitioners
+  // require this.  We are using the GRAIN_SIZE=1 from MooseMesh.C,
+  // not sure how this value was decided upon.
+  //
+  // Note: we don't have to invalidate/update as much stuff as
+  // DisplacedProblem::updateMesh() does, since this will be handled
+  // by a later call to updateMesh().
+  NodeRange node_range(_mesh.getMesh().nodes_begin(),
+                       _mesh.getMesh().nodes_end(),
+                       /*grainsize=*/1);
+
+  // Undisplace the mesh using threads.
+  Threads::parallel_for (node_range, UpdateDisplacedMeshThread(*this));
+}

framework/src/base/UpdateDisplacedMeshThread.C

@@ -87,3 +87,21 @@ UpdateDisplacedMeshThread::operator() (const SemiLocalNodeRange & range) const
     }
   }
 }
+
+void
+UpdateDisplacedMeshThread::operator() (const NodeRange & range) const
+{
+  NodeRange::const_iterator nd = range.begin();
+
+  for (; nd != range.end(); ++nd)
+  {
+    Node & displaced_node = **nd;
+
+    // Get the same node from the reference mesh.
+    Node & reference_node = _ref_mesh.node(displaced_node.id());
+
+    // Undisplace the node
+    for (unsigned int i=0; i<LIBMESH_DIM; ++i)
+      displaced_node(i) = reference_node(i);
+  }
+}