Index: ThirdParty/mpqc_open/src/lib/util/group/LIBS.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/LIBS.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/LIBS.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,7 @@
+libSCgroup.LIBSUF
+#include <util/state/LIBS.h>
+#include <util/keyval/LIBS.h>
+#include <util/misc/LIBS.h>
+#include <util/class/LIBS.h>
+#include <util/container/LIBS.h>
+#include <util/ref/LIBS.h>
Index: ThirdParty/mpqc_open/src/lib/util/group/Makefile
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/Makefile	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/Makefile	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,142 @@
+#
+# Makefile
+#
+# Copyright (C) 1996 Limit Point Systems, Inc.
+#
+# Author: Curtis Janssen <cljanss@ca.sandia.gov>
+# Maintainer: LPS
+#
+# This file is part of the SC Toolkit.
+#
+# The SC Toolkit is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Library General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# The SC Toolkit is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU Library General Public License
+# along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+# the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+# The U.S. Government is granted a limited license as per AL 91-7.
+#
+
+TOPDIR=../../../..
+ifndef SRCDIR
+  SRCDIR=$(shell pwd)
+endif
+include $(SRCDIR)/$(TOPDIR)/lib/GlobalMakefile
+
+REQUIREDCXXSRC = mstate.cc \
+         message.cc messimpl.cc messproc.cc messint.cc \
+         reduce.cc topology.cc hcube.cc \
+         rnglock.cc memory.cc memmsg.cc pool.cc \
+         memamsg.cc memproc.cc \
+         pregtime.cc thread.cc memiter.cc memrdma.cc
+
+OPTIONALCXXSRC = messshm.cc messmpi.cc memshm.cc \
+                 memmtmpi.cc
+
+CXXSRC := $(REQUIREDCXXSRC)
+
+ifeq ($(HAVE_SYSV_IPC),yes)
+  CXXSRC += messshm.cc memshm.cc globcnt.cc
+endif
+
+ifeq ($(HAVE_MPI),yes)
+  CXXSRC += messmpi.cc memmtmpi.cc
+endif
+ifeq ($(HAVE_ARMCI),yes)
+  CXXSRC += memarmci.cc
+endif
+ifeq ($(HAVE_PTHREAD),yes)
+  CXXSRC += thpthd.cc
+endif
+
+LIBOBJ = $(CXXSRC:%.cc=%.$(OBJSUF))
+
+BIN_OR_LIB = LIB
+TARGET_TO_MAKE = libSCgroup
+
+TESTCXXSRC = pooltest.cc messtest.cc rnglocktest.cc thrtest.cc prttest.cc
+DISTFILES = $(REQUIREDCXXSRC) $(OPTIONALCXXSRC) $(INC) \
+            Makefile LIBS.h $(TESTCXXSRC)
+
+TESTPROGS = pooltest messtest rnglocktest memtest
+
+DEPENDINCLUDE =
+
+default:: $(DEPENDINCLUDE)
+
+memtest.$(OBJSUF): memtest.cc
+	$(LTCOMP) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -DSRCDIR=\"$(SRCDIR)\" -c $<
+
+prttest: prttest.$(OBJSUF) \
+	libSCgroup.$(LIBSUF) \
+	libSCkeyval.$(LIBSUF) \
+	libSCmisc.$(LIBSUF) \
+	libSCstate.$(LIBSUF) \
+	libSCclass.$(LIBSUF) \
+	libSCcontainer.$(LIBSUF) \
+	libSCref.$(LIBSUF)
+	$(LTLINK) $(LD) $(LDFLAGS) -o prttest $^ $(SYSLIBS) $(LTLINKBINOPTS)
+
+memtest: memtest.$(OBJSUF) \
+	libSCgroup.$(LIBSUF) \
+	libSCkeyval.$(LIBSUF) \
+	libSCmisc.$(LIBSUF) \
+	libSCstate.$(LIBSUF) \
+	libSCclass.$(LIBSUF) \
+	libSCcontainer.$(LIBSUF) \
+	libSCref.$(LIBSUF)
+	$(LTLINK) $(LD) $(LDFLAGS) -o memtest $^ $(SYSLIBS) $(LTLINKBINOPTS)
+
+pooltest: pooltest.$(OBJSUF) \
+	libSCmisc.$(LIBSUF) \
+	libSCgroup.$(LIBSUF) \
+	libSCkeyval.$(LIBSUF) \
+	libSCstate.$(LIBSUF) \
+	libSCclass.$(LIBSUF) \
+	libSCcontainer.$(LIBSUF) \
+	libSCref.$(LIBSUF)
+	$(LTLINK) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o pooltest $^ $(SYSLIBS) $(LTLINKBINOPTS)
+
+rnglocktest: rnglocktest.$(OBJSUF) libSCmisc.$(LIBSUF) libSCgroup.$(LIBSUF)
+	$(LTLINK) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o rnglocktest $^ $(SYSLIBS) $(LTLINKBINOPTS)
+
+messtest.$(OBJSUF): messtest.cc
+	$(LTCOMP) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -DSRCDIR=\"$(SRCDIR)\" -c $<
+
+messtest: messtest.$(OBJSUF) \
+	libSCgroup.$(LIBSUF) \
+	libSCkeyval.$(LIBSUF) \
+	libSCmisc.$(LIBSUF) \
+	libSCstate.$(LIBSUF) \
+	libSCclass.$(LIBSUF) \
+	libSCcontainer.$(LIBSUF) \
+	libSCref.$(LIBSUF)
+	$(LTLINK) $(LD) $(LDFLAGS) -o messtest $^ $(SYSLIBS) $(LTLINKBINOPTS)
+
+thrtest.$(OBJSUF): thrtest.cc
+	$(LTCOMP) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -DSRCDIR=\"$(SRCDIR)\" -c $<
+
+thrtest: thrtest.$(OBJSUF) \
+	libSCgroup.$(LIBSUF) \
+	libSCkeyval.$(LIBSUF) \
+	libSCmisc.$(LIBSUF) \
+	libSCstate.$(LIBSUF) \
+	libSCclass.$(LIBSUF) \
+	libSCcontainer.$(LIBSUF) \
+	libSCref.$(LIBSUF)
+	$(LTLINK) $(LD) $(LDFLAGS) -o thrtest $^ $(SYSLIBS) $(LTLINKBINOPTS)
+
+include $(SRCDIR)/$(TOPDIR)/lib/GlobalRules
+
+$(TESTCXXSRC:%.cc=%.d) $(LIBOBJ:.$(OBJSUF)=.d): $(DEPENDINCLUDE)
+ifneq ($(DODEPEND),no)
+include $(TESTCXXSRC:%.cc=%.d) $(LIBOBJ:.$(OBJSUF)=.d)
+endif
Index: ThirdParty/mpqc_open/src/lib/util/group/file.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/file.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/file.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,349 @@
+//
+// file.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#include <sstream>
+#include <util/misc/string.h>
+
+#ifdef HAVE_CONFIG_H
+#include <scconfig.h>
+#endif
+
+
+#include <util/misc/formio.h>
+#include <util/class/scexception.h>
+#include <util/group/file.h>
+
+#include <util/group/fileproc.h>
+#if defined(HAVE_MPI)
+#  include <util/group/messmpi.h>
+//#  include <util/group/filemtmpi.h>
+#endif
+
+using namespace std;
+using namespace sc;
+
+//////////////////////////////////////////////////////////////////////
+// FileGrp members
+
+static ClassDesc FileGrp_cd(
+  typeid(FileGrp),"FileGrp",1,"public DescribedClass",
+  0, 0, 0);
+
+FileGrp::FileGrp()
+{
+  debug_ = 0;
+
+  datafile_ = 0;
+  filename_ = 0;
+  offsets_ = 0;
+
+  init_locks();
+}
+
+FileGrp::FileGrp(const Ref<KeyVal>& keyval)
+{
+  debug_ = keyval->intvalue("debug");
+
+  datafile_ = 0;
+  filename_ = 0;
+  offsets_ = 0;
+
+  init_locks();
+}
+
+FileGrp::~FileGrp()
+{
+  delete[] offsets_;
+  delete[] locks_;
+  datafile_ = 0;
+  filename_ = 0;
+}
+
+void
+FileGrp::open()
+{
+  if (filename_) {
+    datafile_ = open(filename_, O_RDWR);
+  }
+  else {
+    throw ProgrammingError("open() called but filename has not been set",
+                           __FILE__, __LINE__, class_desc());
+  }
+}
+
+void
+FileGrp::close()
+{
+  close(datafile_);
+}
+
+void
+FileGrp::set_filename(char *name)
+{
+  if (filename_) {
+    throw ProgrammingError("set_filename() called but filename has been set already",
+                           __FILE__, __LINE__, class_desc());
+  }
+  else
+    filename_ = strdup(name);
+}
+
+const char*
+FileGrp::get_filename() const
+{
+  return filename_;
+}
+
+void
+FileGrp::init_locks()
+{
+  Ref<ThreadGrp> thgrp = ThreadGrp::get_default_threadgrp();
+  nlock_ = 2 * thgrp->nthread();
+  locks_ = new Ref<ThreadLock>[nlock_];
+  for (int i=0; i<nlock_; i++) locks_[i] = thgrp->new_lock();
+}
+
+FileGrp *
+FileGrp::initial_filegrp()
+{
+  int argc = 0;
+  return initial_filegrp(argc,0);
+}
+
+FileGrp *
+FileGrp::initial_filegrp(int &argc, char *argv[])
+{
+  FileGrp *grp = 0;
+
+  char *keyval_string = 0;
+
+  // see if a file group is given on the command line
+  if (argc && argv) {
+      for (int i=0; i<argc; i++) {
+	  if (argv[i] && !strcmp(argv[i], "-filegrp")) {
+              char *filegrp_string = argv[i];
+              i++;
+              if (i >= argc) {
+                  ExEnv::errn() << "-filegrp must be following by an argument"
+                       << endl;
+                  abort();
+                }
+              keyval_string = argv[i];
+              // move the filegrp arguments to the end of argv
+              int j;
+              for (j=i+1; j<argc; j++) {
+                  argv[j-2] = argv[j];
+                }
+              argv[j++] = filegrp_string;
+              argv[j++] = keyval_string;
+              // decrement argc to hide the last two arguments
+              argc -= 2;
+              break;
+            }
+        }
+    }
+
+  if (!keyval_string) {
+      // find out if the environment gives the containing file group
+      keyval_string = getenv("FILEGRP");
+      if (keyval_string) {
+          if (!strncmp("FILEGRP=", keyval_string, 11)) {
+              keyval_string = strchr(keyval_string, '=');
+            }
+          if (*keyval_string == '=') keyval_string++;
+        }
+    }
+
+  // if keyval input for a file group was found, then
+  // create it.
+  if (keyval_string) {
+      //ExEnv::outn() << "Creating FileGrp from \"" << keyval_string << "\"" << endl;
+      Ref<ParsedKeyVal> strkv = new ParsedKeyVal();
+      strkv->parse_string(keyval_string);
+      Ref<DescribedClass> dc = strkv->describedclassvalue();
+      grp = dynamic_cast<FileGrp*>(dc.pointer());
+      if (dc.null()) {
+          ExEnv::errn() << "initial_filegrp: couldn't find a FileGrp in "
+               << keyval_string << endl;
+          abort();
+        }
+      else if (!grp) {
+          ExEnv::errn() << "initial_filegrp: wanted FileGrp but got "
+               << dc->class_name() << endl;
+          abort();
+        }
+      // prevent an accidental delete
+      grp->reference();
+      strkv = 0;
+      dc = 0;
+      // accidental delete not a problem anymore since all smart pointers
+      // to grp are dead
+      grp->dereference();
+      return grp;
+    }
+
+  return grp;
+}
+
+void
+FileGrp::activate()
+{
+}
+
+void
+FileGrp::deactivate()
+{
+}
+
+void
+FileGrp::print(ostream&o) const
+{
+  o << scprintf("FileGrp (node %d):\n", me());
+  o << scprintf("%d: n = %d\n", me(), n());
+  for (int i=0; i<=n_; i++) {
+      o << scprintf("%d: offset[%d] = %5d\n", me(), i, offsets_[i]);
+    }
+}
+
+void
+FileGrp::sum_reduction(double *data, distsize_t doffset, int dlength)
+{
+  distsize_t offset = doffset * sizeof(double);
+  int length = dlength * sizeof(double);
+
+  if (offset + length > totalsize()) {
+      ExEnv::errn() << scprintf("FileGrp::sum_reduction: arg out of range\n");
+      abort();
+    }
+
+  double *source_data = (double*) obtain_readwrite(offset, length);
+
+  for (int i=0; i<dlength; i++) {
+      source_data[i] += data[i];
+    }
+
+  release_readwrite((void*) source_data, offset, length);
+}
+
+void
+FileGrp::sum_reduction_on_node(double *data, size_t doffset, int dlength,
+                                 int node)
+{
+  if (node == -1) node = me();
+
+  sum_reduction(data, doffset + offset(node)/sizeof(double),
+                dlength);
+}
+
+void
+FileGrp::catchup()
+{
+  return;
+}
+
+void
+FileGrp::obtain_local_lock(size_t start, size_t fence)
+{
+  distsize_t locked_region_size = 1 + localsize()/nlock_;
+  int lstart = start/locked_region_size;
+  int llast = fence/locked_region_size;
+  for (int i=lstart; i<=llast; i++) {
+      locks_[i]->lock();
+    }
+}
+
+void
+FileGrp::release_local_lock(size_t start, size_t fence)
+{
+  distsize_t locked_region_size = 1 + localsize()/nlock_;
+  int lstart = start/locked_region_size;
+  int llast = fence/locked_region_size;
+  for (int i=lstart; i<=llast; i++) {
+      locks_[i]->unlock();
+    }
+}
+
+static Ref<FileGrp> default_filegrp;
+
+void
+FileGrp::set_default_filegrp(const Ref<FileGrp>& grp)
+{
+  default_filegrp = grp;
+}
+
+FileGrp*
+FileGrp::get_default_filegrp()
+{
+  if (default_filegrp.nonnull()) return default_filegrp.pointer();
+
+  Ref<MessageGrp> msg = MessageGrp::get_default_messagegrp();
+
+#if defined(HAVE_MPI) && defined(DEFAULT_MTMPI)
+  Ref<ThreadGrp> thr = ThreadGrp::get_default_threadgrp();
+//  default_filegrp = new MTMPIFileGrp(msg,thr);
+  return default_filegrp.pointer();
+#endif
+
+
+  if (msg.null()) {
+      ExEnv::errn() << scprintf("FileGrp::get_default_filegrp: requires default MessageGrp if default behavior not configured\n");
+      abort();
+    }
+#if defined(HAVE_MPI)
+  else if (msg->class_desc() == ::class_desc<MPIMessageGrp>()) {
+      Ref<ThreadGrp> thr = ThreadGrp::get_default_threadgrp();
+//      default_filegrp = new MTMPIFileGrp(msg,thr);
+      return default_filegrp.pointer();
+    }
+#endif
+  else if (msg->n() == 1) {
+      default_filegrp = new ProcFileGrp();
+      return default_filegrp.pointer();
+    }
+  else {
+      ExEnv::errn() << scprintf("FileGrp::get_default_filegrp: cannot create "
+              "default for \"%s\"\n.", msg->class_name());
+      abort();
+    }
+
+  if (default_filegrp.null()) {
+      ExEnv::err0() << scprintf("WARNING: FileGrp::get_default_filegrp(): failed\n");
+      default_filegrp = new ProcFileGrp;
+    }
+  return default_filegrp.pointer();
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/file.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/file.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/file.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,181 @@
+//
+// file.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_file_h
+#define _util_group_file_h
+
+#include <iostream>
+
+#include <scconfig.h>
+#include <util/class/class.h>
+#include <util/group/thread.h>
+#include <util/group/memory.h>
+
+namespace sc {
+
+/** The FileGrp abstract class provides a way of accessing distributed
+file in a parallel machine.  Several specializations are available.  For
+one processor, ProcFileGrp provides a simple stub implementation.
+Otherwise, the specializations that should work are MPIIOIFileGrp and MTMPIFileGrp.
+
+If a FileGrp is not given to the program, then one will be automatically
+chosen depending on which MessageGrp is used by default, the type of
+machine on which the code was compiled, and what options were given at
+configuration time.
+*/
+
+class FileGrp: public DescribedClass {
+  private:
+    int datafile_;
+    char *filename_;
+
+    Ref<ThreadLock> *locks_;
+    int nlock_;
+ 
+    void init_locks();
+
+
+  protected:
+    
+    // derived classes must fill in all these
+    // ~FileGrp deletes the arrays
+    int me_;
+    int n_;
+    distsize_t *offsets_; // offsets_[n_] is the fence for all data
+
+    // set to nonzero for debugging information
+    int debug_;
+
+    void obtain_local_lock(size_t start, size_t fence);
+    void release_local_lock(size_t start, size_t fence);
+  public:
+    FileGrp();
+    FileGrp(const Ref<KeyVal>&);
+    virtual ~FileGrp();
+
+    /// Opens the files
+    void open();
+    /// Closes the files
+    void close();
+    /// Sets the filename for the FileGrp
+    void set_filename(char *name);
+    /// Returns the filename for the FileGrp
+    const char* get_filename() const { return datafile_; };
+    
+    /// Returns who I am.
+    int me() const { return me_; }
+    /// Returns how many nodes there are.
+    int n() const { return n_; }
+
+    /** Set the size of locally held data.
+        When data is accessed using a global offset counting
+        starts at node 0 and proceeds up to node n() - 1. */
+    virtual void set_localsize(size_t) = 0;
+    /// Returns the amount of data residing locally on me().
+    size_t localsize() { return distsize_to_size(offsets_[me_+1]-offsets_[me_]); }
+    /// Returns the global offset to this node's data.
+    distsize_t localoffset() { return offsets_[me_]; }
+    /// Returns the amount of data residing on node.
+    int size(int node)
+        { return distsize_to_size(offsets_[node+1] - offsets_[node]); }
+    /// Returns the global offset to node's data.
+    distsize_t offset(int node) { return offsets_[node]; }
+    /// Returns the sum of all data allocated on all nodes.
+    distsize_t totalsize() { return offsets_[n_]; }
+
+    /// Activate is called before the data is to be used.
+    virtual void activate();
+    /// Deactivate is called after the data has been used.
+    virtual void deactivate();
+
+    /// This gives write access to the data location.  No locking is done.
+    virtual void *obtain_writeonly(distsize_t offset, int size) = 0;
+    /** Only one thread can have an unreleased obtain_readwrite at a time.
+        The actual file region locked can be larger than that requested.
+        If the file region is already locked this will block.  For this
+        reason, data should be held as read/write for as short a time as
+        possible. */
+    virtual void *obtain_readwrite(distsize_t offset, int size) = 0;
+    /// This gives read access to the file location.  No locking is done.
+    virtual void *obtain_readonly(distsize_t offset, int size) = 0;
+    /// This is called when read access is no longer needed.
+    virtual void release_readonly(void *data, distsize_t offset, int size) = 0;
+    /// This is called when write access is no longer needed.
+    virtual void release_writeonly(void *data, distsize_t offset, int size)=0;
+    /** This is called when read/write access is no longer needed.
+        The data will be unlocked. */
+    virtual void release_readwrite(void *data, distsize_t offset, int size)=0;
+
+    virtual void sum_reduction(double *data, distsize_t doffset, int dsize);
+    virtual void sum_reduction_on_node(double *data, size_t doffset, int dsize,
+                                       int node = -1);
+
+    /** Synchronizes all the nodes.  Consider using this when the way you
+        you access data changes. */
+    virtual void sync() = 0;
+
+    /** Processes outstanding requests. Some file group implementations
+        don't have access to real shared memory or even active messages.
+        Instead, requests are processed whenever certain file group
+        routines are called.  This can cause large latencies and buffer
+        overflows.  If this is a problem, then the catchup member can be
+        called to process all outstanding requests. */
+    virtual void catchup();
+
+    /// Prints out information about the object.
+    virtual void print(std::ostream &o = ExEnv::out0()) const;
+
+    /** Create a file group.  This routine looks for a -filegrp
+        argument, then the environmental variable FILEGRP, and, finally,
+        the default MessageGrp object to decide which specialization of
+        FileGrp would be appropriate.  The argument to -integralgrp should
+        be either string for a ParsedKeyVal constructor or a classname.
+        The default ThreadGrp and MessageGrp objects should be initialized
+        before this is called. */
+    static FileGrp* initial_filegrp(int &argc, char** argv);
+    static FileGrp* initial_filegrp();
+    /** The default file group contains the primary file group to
+        be used by an application. */
+    static void set_default_filegrp(const Ref<FileGrp>&);
+    /// Returns the default file group.
+    static FileGrp* get_default_filegrp();
+    /// Clones the given FileGrp. The new FileGrp may need to be initialized additionally.
+    virtual FileGrp* clone() =0;
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/fileproc.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/fileproc.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/fileproc.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,133 @@
+//
+// fileproc.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_fileproc_cc
+#define _util_group_fileproc_cc
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#include <util/group/fileproc.h>
+
+using namespace sc;
+
+static ClassDesc ProcFileGrp_cd(
+  typeid(ProcFileGrp),"ProcFileGrp",1,"public FileGrp",
+  0, create<ProcFileGrp>, 0);
+
+ProcFileGrp::ProcFileGrp()
+{
+  data_ = 0;
+  offsets_ = 0;
+}
+
+ProcFileGrp::ProcFileGrp(const Ref<KeyVal>& keyval):
+  FileGrp(keyval)
+{
+  data_ = 0;
+  offsets_ = 0;
+}
+
+ProcFileGrp::~ProcFileGrp()
+{
+  delete[] data_;
+}
+
+ProcFileGrp*
+ProcFileGrp::clone()
+{
+  return new ProcFileGrp;
+}
+
+void
+ProcFileGrp::set_localsize(size_t localsize)
+{
+  delete[] offsets_;
+  delete[] data_;
+  offsets_ = new distsize_t[2];
+  offsets_[0] = 0;
+  offsets_[1] = localsize;
+  n_ = 1;
+  me_ = 0;
+  data_ = new char[localsize];
+}
+
+void *
+ProcFileGrp::localdata()
+{
+  return data_;
+}
+
+void *
+ProcFileGrp::obtain_readwrite(distsize_t offset, int size)
+{
+  obtain_local_lock(offset-localoffset(), offset-localoffset()+size);
+  return &data_[distsize_to_size(offset)];
+}
+
+void *
+ProcFileGrp::obtain_readonly(distsize_t offset, int size)
+{
+  return &data_[distsize_to_size(offset)];
+}
+
+void *
+ProcFileGrp::obtain_writeonly(distsize_t offset, int size)
+{
+  return &data_[distsize_to_size(offset)];
+}
+
+void
+ProcFileGrp::release_readonly(void *data, distsize_t offset, int size)
+{
+}
+
+void
+ProcFileGrp::release_writeonly(void *data, distsize_t offset, int size)
+{
+}
+
+void
+ProcFileGrp::release_readwrite(void *data, distsize_t offset, int size)
+{
+  release_local_lock(offset-localoffset(), offset-localoffset()+size);
+}
+
+void
+ProcFileGrp::sync()
+{
+}
+
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/fileproc.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/fileproc.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/fileproc.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,73 @@
+//
+// fileproc.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_fileproc_h
+#define _util_group_fileproc_h
+
+#include <sys/types.h>
+
+#include <util/group/memmsg.h>
+
+namespace sc {
+
+/** The ProcFileGrp concrete class provides an implementation of
+FileGrp for a single processor. */
+class ProcFileGrp: public FileGrp {
+  private:
+    int datafile_;
+  public:
+    ProcFileGrp();
+    ProcFileGrp(const Ref<KeyVal>&);
+    ~ProcFileGrp();
+
+    ProcFileGrp* clone();
+    
+    void set_localsize(size_t);
+    void *localdata();
+
+    void *obtain_readwrite(distsize_t offset, int size);
+    void *obtain_readonly(distsize_t offset, int size);
+    void *obtain_writeonly(distsize_t offset, int size);
+    void release_readonly(void *data, distsize_t offset, int size);
+    void release_writeonly(void *data, distsize_t offset, int size);
+    void release_readwrite(void *data, distsize_t offset, int size);
+
+    void sync();
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/globcnt.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/globcnt.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/globcnt.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,182 @@
+//
+// globcnt.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUG__
+#pragma implementation
+#endif
+
+#ifdef HAVE_CONFIG_H
+#include <scconfig.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/ipc.h>
+#include <sys/sem.h>
+#include <util/group/globcnt.h>
+
+using namespace sc;
+
+#ifndef SEM_A
+#  define SEM_A 0200
+#endif
+
+#ifndef SEM_R
+#  define SEM_R 0400
+#endif
+
+GlobalCounter::GlobalCounter()
+{
+  semid_ = -1;
+  controls_release_ = 0;
+}
+
+void
+GlobalCounter::cleanup()
+{
+  if (semid_ != -1 && controls_release_) {
+      int ret;
+#ifdef SEMCTL_REQUIRES_SEMUN
+      semun junk;
+      junk.val = 0;
+#else
+      int junk = 0;
+#endif
+      ret = semctl(semid_, 0, IPC_RMID, junk);
+      if (ret == -1) {
+          perror("semctl (IPC_RMID)");
+          abort();
+        }
+
+      semid_ = -1;
+    }
+}
+
+void
+GlobalCounter::initialize()
+{
+  cleanup();
+  semid_ = semget(IPC_PRIVATE, 1, IPC_CREAT | SEM_R | SEM_A );
+  if (semid_ == -1) {
+      perror("semget");
+      abort();
+    }
+  controls_release_ = 1;
+  operator = (0);
+}
+
+void
+GlobalCounter::initialize(const char *stringrep)
+{
+  semid_ = atoi(stringrep);
+  controls_release_ = 0;
+}
+
+GlobalCounter::~GlobalCounter()
+{
+  cleanup();
+}
+
+void
+GlobalCounter::operator = (int i)
+{
+#ifdef SEMCTL_REQUIRES_SEMUN
+  semun val;
+  val.val = i;
+#else
+  int val = i;
+#endif
+  if (semctl(semid_, 0, SETVAL, val) == -1) {
+      perror("semctl (SETVAL)");
+      abort();
+    }
+}
+
+int
+GlobalCounter::val()
+{
+#ifdef SEMCTL_REQUIRES_SEMUN
+  semun val;
+  val.val = 0;
+#else
+  int val = 0;
+#endif
+  int ret;
+  if ((ret = semctl(semid_, 0, GETVAL, val)) == -1) {
+      perror("semctl (GETVAL)");
+      abort();
+    }
+  return ret;
+}
+
+void
+GlobalCounter::wait_for_zero()
+{
+  operator += (0);
+}
+
+void
+GlobalCounter::operator+=(int i)
+{
+  struct sembuf s;
+  s.sem_num = 0;
+  s.sem_op = i;
+  s.sem_flg = 0;
+  if (semop(semid_, &s, 1) == -1) {
+      perror("semop");
+      abort();
+    }
+}
+
+void
+GlobalCounter::operator--()
+{
+  operator += (-1);
+}
+
+void
+GlobalCounter::operator++()
+{
+  operator += (1);
+}
+
+char *
+GlobalCounter::stringrep()
+{
+  char tmp[80];
+  sprintf(tmp, "%d", semid_);
+  return strcpy(new char[strlen(tmp)+1], tmp);
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/globcnt.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/globcnt.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/globcnt.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,76 @@
+//
+// globcnt.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_globcnt_h
+#define _util_group_globcnt_h
+
+namespace sc {
+
+// A process can create a GlobalCounter using the void CTOR.
+// This process can share the string representation of the
+// counter with other processes.  They can then use the const
+// char * CTOR to create global counters that reference the
+// same global counter.
+
+class GlobalCounter {
+  private:
+    int semid_;
+    int controls_release_;
+
+    void cleanup();
+    
+  public:
+    GlobalCounter();
+    void initialize();
+    void initialize(const char *stringrep);
+    ~GlobalCounter();
+
+    char *stringrep();
+
+    void wait_for_zero();
+    void operator += (int);
+    void operator ++();
+    void operator --();
+    void operator ++(int) { operator++(); }
+    void operator --(int) { operator--(); }
+    void operator = (int);
+
+    int val();
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/group.dox
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/group.dox	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/group.dox	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,74 @@
+
+/** \page group The Group Library
+
+The group library provides three mechanisms for parallel execution and
+communication.  These are:
+
+<ul>
+  <li> \ref messagepassing
+  <li> \ref distmemory
+  <li> \ref multithreading
+</ul>
+
+\section messagepassing Message Passing
+
+The MessageGrp class is the ancestor for classes that provide the passing
+of messages between processes.  There are three important specializations
+of this class:
+
+<dl>
+
+  <dt>ProcMessageGrp<dd>Provides a dummy MessageGrp for single processor
+                        case.
+
+  <dt>MPIMessageGrp<dd>Implements MessageGrp using the Message Passing
+                       Interface (MPI) library.
+
+  <dt>ShmMessageGrp<dd>Implements MessageGrp using SysV Interprocess
+                       communication.
+
+</dl>
+
+Due to the widespread acceptance of MPI and the large number of features
+that it supports, the MessageGrp specialization may one day be eliminated.
+
+\section distmemory Distributed Shared Memory
+
+The MemoryGrp class is the ancestor for classes that permit access to the
+memory in different processes, possible on difference machines.  There are
+two important specializations of this class:
+
+<dl>
+
+  <dt>ProcMemoryGrp<dd>Provides a dummy MemoryGrp for single processor
+                       case.
+
+  <dt>MTMPIMemoryGrp<dd> This works reliably and efficiently, however,
+                         requires POSIX threads and a thread-safe MPI
+                         implementation.
+
+  <dt>ARMCIMemoryGrp<dd> This message group uses the Aggregate Remote Copy
+                         Interface (ARMCI).  This exploits the RDMA
+                         capabilities of interconnects such as Myrinet and
+                         InfiniBand.
+
+
+</dl>
+
+\section multithreading Multi-Threading
+
+The ThreadGrp class is the ancestor of classes that provide
+multiple-threads within a single address space.  There are
+two important specializations of this class:
+
+<dl>
+
+  <dt>ProcThreadGrp<dd>Provides a dummy ThreadGrp for the single thread
+                       case.
+
+  <dt>PthreadThreadGrp<dd> This is a ThreadGrp that uses POSIX threads.
+
+</dl>
+
+
+*/
Index: ThirdParty/mpqc_open/src/lib/util/group/hcube.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/hcube.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/hcube.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,160 @@
+//
+// hcube.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#include <util/group/message.h>
+#include <util/group/topology.h>
+#include <util/group/hcube.h>
+
+using namespace sc;
+
+static ClassDesc HypercubeGMI_cd(
+  typeid(HypercubeGMI),"HypercubeGMI",1,"public GlobalMsgIter",
+  0, 0, 0);
+
+HypercubeGMI::HypercubeGMI(int nproc, int me, int root):
+  GlobalMsgIter(nproc, me, root)
+{
+  int i;
+
+  // compute the number of steps needed
+  i = nproc;
+  n_ = 0;
+  while(i>1) {
+      i = i>>1;
+      n_++;
+    }
+  if (1<<n_ != nproc) n_++;
+  if (root != 0) n_++;
+
+  // compute the size of the hypercube that embeds these processors
+  nhyper_ = 1;
+  while (nhyper_ < nproc) nhyper_ = nhyper_ << 1;
+}
+
+HypercubeGMI::~HypercubeGMI()
+{
+}
+
+int
+HypercubeGMI::fwdsendto()
+{
+  int offset;
+  if (root_ != 0) {
+      if (i_ == 0) {
+          if (root_ == me_) return 0;
+          return -1;
+        }
+      else offset = 1;
+    }
+  else offset = 0;
+
+  int bit = 1<<(i_-offset);
+  int highbits = (nhyper_-2)<<(i_-offset);
+  // if i don't have this bit and none above, then i'm a sender
+  if (!(me_&bit) && !(me_&highbits)) {
+      int target = me_ + bit;
+      if (target >= nproc_) return -1;
+      // already got this one
+      if (target == root_) return -1;
+      return target;
+    }
+  return -1;
+}
+
+int
+HypercubeGMI::fwdsend()
+{
+  return fwdsendto() != -1;
+}
+
+int
+HypercubeGMI::fwdrecvfrom()
+{
+  int offset;
+  if (root_ != 0) {
+      if (i_ == 0) {
+          if (me_ == 0) return root_;
+          return -1;
+        }
+      else offset = 1;
+    }
+  else offset = 0;
+
+  // already got this one
+  if (me_ == root_) return -1;
+
+  int bit = 1<<(i_-offset);
+  int highbits = (nhyper_-2)<<(i_-offset);
+  if (!(me_&bit) && !(me_&highbits)) {
+      return -1;
+    }
+  else if (!(me_&highbits)) {
+      int source = me_ - bit;
+      return source;
+    }
+  return -1;
+}
+
+int
+HypercubeGMI::fwdrecv()
+{
+  return fwdrecvfrom() != -1;
+}
+
+///////////////////////////////////////////////////////////////////////////
+// HypercubeTopology members
+
+static ClassDesc HypercubeTopology_cd(
+  typeid(HypercubeTopology),"HypercubeTopology",1,"public MachineTopology",
+  0, create<HypercubeTopology>, 0);
+
+HypercubeTopology::HypercubeTopology()
+{
+}
+
+HypercubeTopology::HypercubeTopology(const Ref<KeyVal>& keyval):
+  MachineTopology(keyval)
+{
+}
+
+HypercubeTopology::~HypercubeTopology()
+{
+}
+
+Ref<GlobalMsgIter>
+HypercubeTopology::global_msg_iter(const Ref<MessageGrp>& grp,
+                                   int root)
+{
+  return new HypercubeGMI(grp->n(), grp->me(), root);
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/hcube.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/hcube.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/hcube.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,65 @@
+//
+// hcube.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_hcube_h
+#define _util_group_hcube_h
+
+#include <util/group/topology.h>
+
+namespace sc {
+
+class HypercubeGMI: public GlobalMsgIter {
+  private:
+    int nhyper_;
+  protected:
+    int fwdsendto();
+    int fwdsend();
+    int fwdrecvfrom();
+    int fwdrecv();
+  public:
+    HypercubeGMI(int nproc, int me, int root);
+    ~HypercubeGMI();
+};
+
+// This utilitizes a hypercube topology, but will work for any number of
+// nodes.
+class HypercubeTopology: public MachineTopology {
+  public:
+    HypercubeTopology();
+    HypercubeTopology(const Ref<KeyVal>&);
+    ~HypercubeTopology();
+    Ref<GlobalMsgIter> global_msg_iter(const Ref<MessageGrp>&, int target);
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/linkage.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/linkage.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/linkage.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,70 @@
+//
+// linkage.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@ca.sandia.gov>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_linkage_h
+#define _util_group_linkage_h
+
+#ifdef HAVE_CONFIG_H
+#include <scconfig.h>
+#endif
+
+#include <util/group/memproc.h>
+
+namespace sc {
+static ForceLink<ProcMessageGrp> group_force_link_0_;
+static ForceLink<ProcMemoryGrp> group_force_link_1_;
+}
+
+# ifdef HAVE_SYSV_IPC
+#   include <util/group/messshm.h>
+namespace sc {
+    static ForceLink<ShmMessageGrp> group_force_link_a_;
+}
+# endif
+
+# if defined(HAVE_PTHREAD)
+#   include <util/group/thpthd.h>
+namespace sc {
+    static ForceLink<PthreadThreadGrp> group_force_link_c_;
+}
+# endif
+
+#if defined(HAVE_MPI)
+#   include <util/group/memmtmpi.h>
+namespace sc {
+    static ForceLink<MTMPIMemoryGrp> group_force_link_g_;
+}
+#endif
+
+#if defined(HAVE_ARMCI)
+#   include <util/group/memarmci.h>
+namespace sc {
+    static ForceLink<ARMCIMemoryGrp> group_force_link_h_;
+}
+#endif
+
+#endif
Index: ThirdParty/mpqc_open/src/lib/util/group/memamsg.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memamsg.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memamsg.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,368 @@
+//
+// memamsg.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_memamsg_cc
+#define _util_group_memamsg_cc
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#include <util/misc/formio.h>
+#include <util/group/pool.h>
+#include <util/group/memamsg.h>
+#include <util/group/memiter.h>
+
+using namespace std;
+using namespace sc;
+
+#ifdef HAVE_HRECV
+#  define DISABLE do { masktrap(1); ExEnv::outn().flush(); } while(0)
+#  define ENABLE do { ExEnv::outn().flush(); masktrap(0); } while(0)
+   extern "C" {
+       long masktrap(long state);
+     }
+#else
+#  define DISABLE ExEnv::outn().flush()
+#  define ENABLE ExEnv::outn().flush()
+#endif
+
+#define PRINTF(args) do { DISABLE; \
+                          ExEnv::outn() << scprintf args ; \
+                          ExEnv::outn().flush(); \
+                          ENABLE; \
+                         } while(0)
+
+#undef PRINTF
+#define PRINTF(args)
+
+///////////////////////////////////////////////////////////////////////
+// The MemoryDataRequest class
+
+MemoryDataRequest::MemoryDataRequest(Request r, int node,
+                                     int offset, int size, int lock,
+                                     int serial_number)
+{
+  assign(r, node, offset, size, lock, serial_number);
+}
+
+void
+MemoryDataRequest::assign(Request r, int node,
+                          int offset, int size, int lock,
+                          int serial_number)
+{
+  data_[0] = (int) r;
+  data_[1] = node;
+  data_[2] = offset;
+  data_[3] = size;
+  data_[4] = serial_number;
+  data_[5] = lock;
+}
+
+const char *
+MemoryDataRequest::request_string() const
+{
+  switch (request()) {
+  case MemoryDataRequest::Deactivate:
+      return "Deactivate";
+  case MemoryDataRequest::Retrieve:
+      return "Retrieve";
+  case MemoryDataRequest::Replace:
+      return "Replace";
+  case MemoryDataRequest::DoubleSum:
+      return "DoubleSum";
+  case MemoryDataRequest::Sync:
+      return "Sync";
+  default:
+      return "BadRequest";
+    }
+}
+
+void
+MemoryDataRequest::print(const char *msg, ostream & o) const
+{
+  if (msg == 0) msg = "";
+
+  o.flush();
+  if (request() == Sync) {
+      o << scprintf("%s \"%s\" %d-%d reactivate = %d\n",
+              msg, request_string(), node(), serial_number(), reactivate());
+    }
+  else {
+      o << scprintf("%s \"%s\" offset = %5d, %5d bytes, %d-%d, %s\n",
+              msg, request_string(),
+              offset(), size(), node(), serial_number(),
+              (lock()?"lock":"nolock"));
+    }
+  o.flush();
+}
+
+void
+MemoryDataRequest::operator =(const MemoryDataRequest &r)
+{
+  for (int i=0; i<NData; i++) {
+      data_[i] = r.data_[i];
+    }
+}
+
+///////////////////////////////////////////////////////////////////////
+// The MemoryDataRequestQueue class
+
+void
+MemoryDataRequestQueue::push(MemoryDataRequest&r)
+{
+  if (n_ == MaxDepth) {
+      ExEnv::errn() << scprintf("MemoryDataRequestQueue: MaxDepth exceeded\n");
+      abort();
+    }
+  q_[n_] = r;
+  n_++;
+}
+
+void
+MemoryDataRequestQueue::pop(MemoryDataRequest&r)
+{
+  if (n_ == 0) {
+      ExEnv::errn() << scprintf("MemoryDataRequestQueue: nothing to pop\n");
+      abort();
+    }
+  n_--;
+  r = q_[n_];
+}
+
+///////////////////////////////////////////////////////////////////////
+// Members for ActiveMsgMemoryGrp
+
+static ClassDesc ActiveMsgMemoryGrp_cd(
+  typeid(ActiveMsgMemoryGrp),"ActiveMsgMemoryGrp",1,"public MsgMemoryGrp",
+  0, 0, 0);
+
+ActiveMsgMemoryGrp::ActiveMsgMemoryGrp(const Ref<MessageGrp>& msg):
+  MsgMemoryGrp(msg)
+{
+  data_ = 0;
+}
+
+ActiveMsgMemoryGrp::ActiveMsgMemoryGrp(const Ref<KeyVal>& keyval):
+  MsgMemoryGrp(keyval)
+{
+  data_ = 0;
+}
+
+void
+ActiveMsgMemoryGrp::set_localsize(size_t localsize)
+{
+  if (debug_) {
+      ExEnv::out0() << "ActiveMsgMemoryGrp::set_localsize(" << localsize << ")" << endl;
+    }
+  deactivate();
+  MsgMemoryGrp::set_localsize(localsize);
+  delete[] data_;
+  data_ = new char[localsize];
+  activate();
+  if (debug_) {
+      ExEnv::out0() << "ActiveMsgMemoryGrp::set_localsize done: offsets:";
+      for (int i=0; i<=n(); i++) {
+          ExEnv::out0() << " " << double(offset(i));
+        }
+      ExEnv::out0() << endl;
+    }
+}
+
+void *
+ActiveMsgMemoryGrp::localdata()
+{
+  return data_;
+}
+
+ActiveMsgMemoryGrp::~ActiveMsgMemoryGrp()
+{
+  deactivate();
+  delete[] data_;
+}
+
+void *
+ActiveMsgMemoryGrp::obtain_writeonly(distsize_t offset, int size)
+{
+  void *data = (void *) new char[size];
+  return data;
+}
+
+void *
+ActiveMsgMemoryGrp::obtain_readwrite(distsize_t offset, int size)
+{
+  PRINTF(("ActiveMsgMemoryGrp::obtain_readwrite entered\n"));
+  void *data = (void *) new char[size];
+  MemoryIter i(data, offsets_, n());
+  for (i.begin(offset, size); i.ready(); i.next()) {
+      if (i.node() == me()) {
+          PRINTF(("ActiveMsgMemoryGrp::obtain_readwrite: local copy\n"));
+          obtain_local_lock(i.offset(), i.offset()+i.size());
+          memcpy(i.data(), &data_[i.offset()], i.size());
+        }
+      else {
+          PRINTF(("ActiveMsgMemoryGrp::obtain_readwrite: node = %d, "
+                  "int offset = %d, int size = %d\n",
+                  i.node(), i.offset()/sizeof(int), i.size()/sizeof(int)));
+          retrieve_data(i.data(), i.node(), i.offset(), i.size(), 1);
+        }
+    }
+  PRINTF(("ActiveMsgMemoryGrp::obtain_readwrite exiting\n"));
+  return data;
+}
+
+void *
+ActiveMsgMemoryGrp::obtain_readonly(distsize_t offset, int size)
+{
+  void *data = (void *) new char[size];
+  PRINTF(("%d: ActiveMsgMemoryGrp::obtain_readonly:"
+          "overall: offset = %d size = %d\n",
+          me(), offset, size));
+  MemoryIter i(data, offsets_, n());
+  for (i.begin(offset, size); i.ready(); i.next()) {
+      PRINTF(("%d: ActiveMsgMemoryGrp::obtain_readonly:working on:"
+              "node = %d offset = %d size = %d\n",
+              me(), i.node(), i.offset(), i.size()));
+      if (i.node() == me()) {
+          PRINTF(("%d: ActiveMsgMemoryGrp::obtain_readonly: local: "
+                  "offset = %d size = %d\n", me(), i.offset(), i.size()));
+          memcpy(i.data(), &data_[i.offset()], i.size());
+        }
+      else {
+          PRINTF(("ActiveMsgMemoryGrp::obtain_readonly: node = %d, "
+                  "int offset = %d, int size = %d\n",
+                  i.node(), i.offset()/sizeof(int), i.size()/sizeof(int)));
+          retrieve_data(i.data(), i.node(), i.offset(), i.size(), 0);
+        }
+    }
+  return data;
+}
+
+void
+ActiveMsgMemoryGrp::sum_reduction(double *data, distsize_t doffset, int dsize)
+{
+  distsize_t offset = doffset * sizeof(double);
+  int size = dsize * sizeof(double);
+  MemoryIter i(data, offsets_, n());
+  for (i.begin(offset, size); i.ready(); i.next()) {
+      if (i.node() == me()) {
+          int chunkdsize = i.size()/sizeof(double);
+          double *chunkdata = (double*) &data_[i.offset()];
+          double *tmp = (double*) i.data();
+          PRINTF(("%d: summing %d doubles from 0x%x to 0x%x\n",
+                  me(), chunkdsize, tmp, chunkdata));
+          obtain_local_lock(i.offset(), i.offset()+i.size());
+          for (int j=0; j<chunkdsize; j++) {
+              *chunkdata++ += *tmp++;
+            }
+          release_local_lock(i.offset(), i.offset()+i.size());
+        }
+      else {
+          sum_data((double*)i.data(), i.node(), i.offset(), i.size());
+        }
+    }
+}
+
+void
+ActiveMsgMemoryGrp::sum_reduction_on_node(double *data, size_t doffset,
+                                          int dlength, int node)
+{
+  if (node == -1) node = me();
+
+  if (node == me()) {
+      double *localdata = (double*) &data_[sizeof(double)*doffset];
+      obtain_local_lock(sizeof(double)*doffset,
+                        sizeof(double)*(doffset+dlength));
+      for (int j=0; j<dlength; j++) {
+          *localdata++ += *data++;
+        }
+      release_local_lock(sizeof(double)*doffset,
+                         sizeof(double)*(doffset+dlength));
+    }
+  else {
+      sum_data(data, node, sizeof(double)*doffset, sizeof(double)*dlength);
+    }
+}
+
+void
+ActiveMsgMemoryGrp::release_readonly(void *data, distsize_t offset, int size)
+{
+  delete[] (char*) data;
+}
+
+void
+ActiveMsgMemoryGrp::release_writeonly(void *data, distsize_t offset, int size)
+{
+  MemoryIter i(data, offsets_, n());
+  for (i.begin(offset, size); i.ready(); i.next()) {
+      if (i.node() == me()) {
+          PRINTF(("ActiveMsgMemoryGrp::release_write: local\n"));
+          PRINTF(("  i.offset() = %d i.data() = 0x%x i.size() = %d\n",
+                  i.offset(), i.data(), i.size()));
+          PRINTF(("  &data_[i.offset()] = 0x%x\n", &data_[i.offset()]));
+          memcpy(&data_[i.offset()], i.data(), i.size());
+        }
+      else {
+          PRINTF(("ActiveMsgMemoryGrp::release_write: node = %d, "
+                  "int offset = %d, int size = %d\n",
+                  i.node(), i.offset()/sizeof(int), i.size()/sizeof(int)));
+          replace_data(i.data(), i.node(), i.offset(), i.size(), 0);
+        }
+    }
+  delete[] (char*) data;
+}
+
+void
+ActiveMsgMemoryGrp::release_readwrite(void *data, distsize_t offset, int size)
+{
+  MemoryIter i(data, offsets_, n());
+  for (i.begin(offset, size); i.ready(); i.next()) {
+      if (i.node() == me()) {
+          memcpy(&data_[i.offset()], i.data(), i.size());
+          release_local_lock(i.offset(), i.offset()+i.size());
+        }
+      else {
+          replace_data(i.data(), i.node(), i.offset(), i.size(), 1);
+        }
+    }
+  delete[] (char*) data;
+}
+
+void
+ActiveMsgMemoryGrp::print(ostream &o) const
+{
+  MemoryGrp::print(o);
+}
+
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memamsg.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memamsg.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memamsg.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,131 @@
+//
+// memamsg.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_memamsg_h
+#define _util_group_memamsg_h
+
+#include <iostream>
+
+#include <util/group/memmsg.h>
+
+namespace sc {
+    
+class MemoryDataRequest {
+  public:
+    enum { NData = 6 };
+    enum Request { Deactivate, Sync, Retrieve, Replace, DoubleSum };
+  private:
+    int data_[NData];
+  public:
+    MemoryDataRequest() {}
+    MemoryDataRequest(Request r, int node = 0, int offset = 0, int size = 0,
+                      int lock = 0, int serial = 0);
+    void assign(Request r, int node, int offset, int size,
+                int lock, int serial);
+    void *data() const { return (void *) data_; }
+    int nbytes() const { return sizeof(int)*NData; }
+
+    const char *request_string() const;
+
+    MemoryDataRequest::Request request() const { return (Request) data_[0]; }
+    int node() const { return data_[1]; }
+    int offset() const { return data_[2]; }
+    int size() const { return data_[3]; }
+    int serial_number() const { return data_[4]; }
+    int lock() const { return data_[5]; }
+
+    int touches_data() const {return request()!=Deactivate&&request()!=Sync;}
+
+    // Sync messages only define one datum besides type and node
+    int reactivate() const { return data_[2]; }
+
+    void operator =(const MemoryDataRequest &r);
+
+    void print(const char* msg = 0, std::ostream & o = ExEnv::out0()) const;
+};
+
+class MemoryDataRequestQueue {
+  public:
+    enum { MaxDepth = 1024 };
+  private:
+    MemoryDataRequest q_[MaxDepth];
+    int n_;
+  public:
+    MemoryDataRequestQueue(): n_(0) {}
+    int n() const { return n_; }
+    void push(MemoryDataRequest&);
+    void pop(MemoryDataRequest&);
+
+    MemoryDataRequest& operator[](int i) { return q_[i]; }
+    void clear() { n_ = 0; }
+};
+
+/** The ActiveMsgMemoryGrp abstract class specializes the MsgMemoryGrp
+class.  It uses active messages to implement global shared memory.  */
+class ActiveMsgMemoryGrp : public MsgMemoryGrp {
+  protected:
+    char *data_;
+
+    virtual void retrieve_data(void *, int node, int offset, int size,
+                               int lock) = 0;
+    virtual void replace_data(void *, int node, int offset, int size,
+                              int unlock) = 0;
+    virtual void sum_data(double *data, int node, int doffset, int dsize) = 0;
+  public:
+    ActiveMsgMemoryGrp(const Ref<MessageGrp>& msg);
+    ActiveMsgMemoryGrp(const Ref<KeyVal>&);
+    ~ActiveMsgMemoryGrp();
+
+    void set_localsize(size_t);
+    void *localdata();
+
+    void *obtain_writeonly(distsize_t offset, int size);
+    void *obtain_readwrite(distsize_t offset, int size);
+    void *obtain_readonly(distsize_t offset, int size);
+    void release_readonly(void *data, distsize_t offset, int size);
+    void release_writeonly(void *data, distsize_t offset, int size);
+    void release_readwrite(void *data, distsize_t offset, int size);
+
+    void sum_reduction(double *data, distsize_t doffset, int dsize);
+    void sum_reduction_on_node(double *data, size_t doffset, int dsize,
+                               int node = -1);
+
+    void print(std::ostream &o = ExEnv::out0()) const;
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memarmci.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memarmci.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memarmci.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,241 @@
+//
+// memarmci.cc
+// based on memshm.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@ca.sandia.gov>
+// Maintainer: SNL
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_memarmci_cc
+#define _util_group_memarmci_cc
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+extern "C" {
+#include <armci.h>
+}
+
+#include <stdexcept>
+
+#include <util/misc/formio.h>
+#include <util/class/scexception.h>
+#include <util/group/memarmci.h>
+
+using namespace sc;
+
+static ClassDesc ARMCIMemoryGrp_cd(
+  typeid(ARMCIMemoryGrp),"ARMCIMemoryGrp",1,"public RDMAMemoryGrp",
+  0, create<ARMCIMemoryGrp>, 0);
+
+ARMCIMemoryGrp::ARMCIMemoryGrp(const Ref<MessageGrp>& msg):
+  RDMAMemoryGrp(msg)
+{
+  init();
+}
+
+ARMCIMemoryGrp::ARMCIMemoryGrp(const Ref<KeyVal>& keyval):
+  RDMAMemoryGrp(keyval)
+{
+  init();
+}
+
+void
+ARMCIMemoryGrp::init()
+{
+  armci_lock_ = ThreadGrp::get_default_threadgrp()->new_lock();
+  //debug_ = 1;
+  all_data_ = 0;
+  ARMCI_Init();
+}
+
+void
+ARMCIMemoryGrp::finalize()
+{
+  set_localsize(0);
+  ARMCI_Finalize();
+}
+
+void
+ARMCIMemoryGrp::set_localsize(size_t localsize)
+{
+  ARMCI_AllFence();
+
+  // this will initialize the offsets_ array
+  RDMAMemoryGrp::set_localsize(localsize);
+
+  if (all_data_) {
+      ARMCI_Free(data_);
+      delete[] all_data_;
+      all_data_ = 0;
+      data_ = 0;
+      ARMCI_Destroy_mutexes();
+    }
+
+  if (localsize == 0) return;
+
+  all_data_ = new void*[n()];
+  int r;
+  r = ARMCI_Malloc(all_data_, localsize);
+  data_ = reinterpret_cast<char*>(all_data_[me()]);
+
+  if (debug_) {
+    for (int i=0; i<n(); i++) {
+      std::cout << me() << ": all_data[" << i
+		<< "] = " << all_data_[i] << std::endl;
+    }
+  }
+
+  ARMCI_Create_mutexes(1);
+}
+
+void
+ARMCIMemoryGrp::retrieve_data(void *data, int node, int offset,
+                              int size, int lock)
+{
+  if (armci_lock_.nonnull()) armci_lock_->lock();
+  if (lock) ARMCI_Lock(0, node);
+  ARMCI_Get(reinterpret_cast<char*>(all_data_[node])+offset, data, size, node);
+  if (armci_lock_.nonnull()) armci_lock_->unlock();
+}
+
+void
+ARMCIMemoryGrp::replace_data(void *data, int node, int offset,
+                             int size, int unlock)
+{
+  if (armci_lock_.nonnull()) armci_lock_->lock();
+  ARMCI_Put(data, reinterpret_cast<char*>(all_data_[node])+offset, size, node);
+  if (unlock) {
+      ARMCI_Fence(node);
+      ARMCI_Unlock(0, node);
+    }
+  if (armci_lock_.nonnull()) armci_lock_->unlock();
+}
+
+void
+ARMCIMemoryGrp::sum_data(double *data, int node, int offset, int size)
+{
+  int doffset = offset/sizeof(double);
+  int dsize = size/sizeof(double);
+
+  void *src = data;
+  void *dst = reinterpret_cast<double*>(all_data_[node])+doffset;
+
+  armci_giov_t acc_dat;
+  acc_dat.src_ptr_array = &src;
+  acc_dat.dst_ptr_array = &dst;
+  acc_dat.bytes = dsize * sizeof(double);
+  acc_dat.ptr_array_len = 1;
+  double scale = 1.0;
+
+  if (debug_) {
+      std::cout << me() << ": summing " << dsize
+                << " doubles from "
+                << (void*)src
+                << " to "
+                << (void*)dst
+                << " on " << node
+                << " (base dest=" << (void*)all_data_[node] << ")"
+                << std::endl;
+      for (int i=0; i<dsize; i++) {
+          std::cout << me() << ": src[" << i << "] = "
+                    << data[i] << std::endl;
+        }
+//        for (int i=0; i<dsize; i++) {
+//            std::cout << me() << ": dst[" << i << "] = "
+//                      << ((double*)(all_data_[node]))[doffset+i]
+//                      << std::endl;
+//          }
+    }
+
+  if (armci_lock_.nonnull()) armci_lock_->lock();
+  // Original code sending all data at once:
+  // ARMCI_AccV(ARMCI_ACC_DBL, &scale, &acc_dat, 1, node);
+  // Hack to send smaller chunks to not overflow buffers in ARMCI:
+  int incr = 32768;
+  for (int i=0; i<size; i+=incr) {
+      void *tsrc = (&(((char*)src)[i]));
+      void *tdst = (&(((char*)dst)[i]));
+      acc_dat.src_ptr_array = &tsrc;
+      acc_dat.dst_ptr_array = &tdst;
+      if (size - i > incr) acc_dat.bytes = incr;
+      else acc_dat.bytes = (size-i);
+      acc_dat.ptr_array_len = 1;
+      ARMCI_AccV(ARMCI_ACC_DBL, &scale, &acc_dat, 1, node);
+    }
+  // Send data all at once using the contiguous routine (which does not exist):
+  // ARMCI_Acc(ARMCI_ACC_DBL, &scale, src, dst, size, node);
+  if (armci_lock_.nonnull()) armci_lock_->unlock();
+}
+
+void
+ARMCIMemoryGrp::sync()
+{
+  ARMCI_Barrier();
+}
+
+void
+ARMCIMemoryGrp::deactivate()
+{
+  // Really, this is still active after deactivate is called.
+  // However, we'll at least make sure that all outstanding
+  // requests are finished.
+  ARMCI_AllFence();
+}
+
+void*
+ARMCIMemoryGrp::malloc_local(size_t nbyte)
+{
+  void* buf = ARMCI_Malloc_local(nbyte);
+  if (buf == NULL)
+    throw MemAllocFailed("malloc_local -- failed to allocate memory",
+                         __FILE__, __LINE__, nbyte, this->class_desc());
+  return buf;
+}
+
+void
+ARMCIMemoryGrp::free_local(void *data)
+{
+  ARMCI_Free_local(data);
+}
+
+ARMCIMemoryGrp::~ARMCIMemoryGrp()
+{
+  finalize();
+}
+
+void
+ARMCIMemoryGrp::print(std::ostream &o) const
+{
+  RDMAMemoryGrp::print(o);
+}
+
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memarmci.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memarmci.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memarmci.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,77 @@
+//
+// memarmci.h
+// based on memshm.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@ca.sandia.gov>
+// Maintainer: SNL
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_memarmci_h
+#define _util_group_memarmci_h
+
+#include <iostream>
+
+#include <util/group/memrdma.h>
+
+namespace sc {
+
+/** The ARMCIMemoryGrp concrete class provides an implementation of
+    MsgMemoryGrp.  It uses the ARMCI interface. */
+class ARMCIMemoryGrp: public RDMAMemoryGrp {
+  private:
+    void **all_data_;
+    void init();
+    void finalize();
+    Ref<ThreadLock> armci_lock_;
+  public:
+    ARMCIMemoryGrp(const Ref<MessageGrp>& msg);
+    ARMCIMemoryGrp(const Ref<KeyVal>&);
+    ~ARMCIMemoryGrp();
+
+    void set_localsize(size_t);
+
+    void retrieve_data(void *, int node, int offset, int size, int lock);
+    void replace_data(void *, int node, int offset, int size, int unlock);
+    void sum_data(double *data, int node, int doffset, int dsize);
+
+    void sync();
+    void deactivate();
+
+    void* malloc_local(size_t nbyte);
+    void free_local(void *data);
+
+    void print(std::ostream &o = ExEnv::out0()) const;
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memiter.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memiter.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memiter.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,118 @@
+//
+// memiter.cc
+//
+// derived from memamsg.cc
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_memiter_cc
+#define _util_group_memiter_cc
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#include <util/group/memiter.h>
+
+using namespace sc;
+
+///////////////////////////////////////////////////////////////////////
+// The MemoryIter class
+
+int
+MemoryIter::local(distsize_t offset, int size, int node)
+{
+  if (offset >= offsets_[node] && offset + size <= offsets_[node+1])
+      return 1;
+  return 0;
+}
+
+MemoryIter::MemoryIter(void *data,
+                       distsize_t *offsets,
+                       int n):
+  offsets_(offsets),
+  n_(n),
+  data_(data),
+  ready_(0)
+{
+}
+
+void
+MemoryIter::begin(distsize_t offset, int size)
+{
+  offset_ = offset;
+  size_ = size;
+
+  current_data_ = (char *) data_;
+
+  distsize_t fence = offset + size;
+
+  for (node_ = 0; node_ < n_; node_++) {
+      if (offset_ < offsets_[node_ + 1]) {
+          current_offset_ = distsize_to_size(offset_ - offsets_[node_]);
+          if (fence <= offsets_[node_ + 1]) {
+              current_size_ = size_;
+            }
+          else {
+              current_size_ = distsize_to_size(offsets_[node_ + 1] - offset_);
+            }
+          ready_ = 1;
+          return;
+        }
+    }
+
+  // couldn't find the requested data, this is probably from an
+  // invalid argument
+  ready_ = 0;
+}
+
+void
+MemoryIter::next()
+{
+  distsize_t fence = offset_ + size_;
+
+  if (fence <= offsets_[node_ + 1]) {
+      ready_ = 0;
+    }
+  else {
+      node_++;
+      current_data_ += current_size_;
+      if (fence <= offsets_[node_ + 1]) {
+          current_size_ = size_ - distsize_to_size(offsets_[node_] - offset_);
+        }
+      else {
+          current_size_ = distsize_to_size(offsets_[node_+1]-offsets_[node_]);
+        }
+      current_offset_ = 0;
+    }
+}
+
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memiter.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memiter.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memiter.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,81 @@
+//
+// memiter.h
+//
+// derived from memasmg.cc
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_memiter_h
+#define _util_group_memiter_h
+
+#include <util/group/memory.h>
+
+namespace sc {
+
+class MemoryIter {
+  private:
+    distsize_t *offsets_;
+    int n_;
+
+    void *data_;
+
+    char *current_data_;
+    int current_size_;
+    int current_offset_;
+    int node_;
+
+    int ready_;
+
+    distsize_t offset_;
+    int size_;
+  public:
+    MemoryIter(void *data, distsize_t *offsets, int n);
+
+    // iteration control
+    void begin(distsize_t offset, int size);
+    int ready() { return ready_; }
+    void next();
+
+    // info about the current piece of data
+    void *data() { return (void*) current_data_; }
+    int node() { return node_; }
+    int offset() { return current_offset_; }
+    int size() { return current_size_; }
+
+    // returns true if all data is local to node
+    int local(distsize_t offset, int size, int node);
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memmsg.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memmsg.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memmsg.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,131 @@
+//
+// memmsg.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_memmsg_cc
+#define _util_group_memmsg_cc
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#include <util/group/memmsg.h>
+
+using namespace std;
+using namespace sc;
+
+static ClassDesc MsgMemoryGrp_cd(
+  typeid(MsgMemoryGrp),"MsgMemoryGrp",1,"public MemoryGrp",
+  0, 0, 0);
+
+MsgMemoryGrp::MsgMemoryGrp(const Ref<MessageGrp> &msg)
+{
+  msg_ = msg;
+  n_ = msg->n();
+  me_ = msg->me();
+}
+
+MsgMemoryGrp::MsgMemoryGrp(const Ref<KeyVal> &keyval):
+  MemoryGrp(keyval)
+{
+  Ref<MessageGrp> msg; msg << keyval->describedclassvalue("message");
+  if (msg.null()) {
+      msg = MessageGrp::get_default_messagegrp();
+    }
+  if (msg.null()) {
+      ExEnv::errn() << "MsgMemoryGrp(const Ref<KeyVal>&): couldn't find MessageGrp"
+           << endl;
+      abort();
+    }
+
+  msg_ = msg;
+  n_ = msg->n();
+  me_ = msg->me();
+}
+
+MsgMemoryGrp::~MsgMemoryGrp()
+{
+}
+
+void
+MsgMemoryGrp::set_localsize(size_t localsize)
+{
+  delete[] offsets_;
+
+  offsets_ = new distsize_t[n_ + 1];
+  int *sizes = new int[n_];
+
+  int i;
+  for (i=0; i<n_; i++) sizes[i] = 0;
+  sizes[me_] = localsize;
+
+  msg_->sum(sizes, n_);
+
+  offsets_[0] = 0;
+  for (i=1; i<=n_; i++) {
+      offsets_[i] = sizes[i-1] + offsets_[i-1];
+      if (offsets_[i] < offsets_[i-1]) {
+          ExEnv::errn() << "MsgMemoryGrp::set_localsize: distsize_t cannot handle biggest size" << endl;
+          abort();
+        }
+    }
+
+  delete[] sizes;
+}
+
+void
+MsgMemoryGrp::sync()
+{
+  msg_->sync();
+#if 0
+  // debug code
+  for (int i=0; i<n(); i++) {
+      if (i == me()) {
+          cout << "Data on node " << i << ":" << endl;
+          for (int j=0; j<localsize()/sizeof(double); j++) {
+              double dat = ((double*)localdata())[j];
+              if (fabs(dat) > 1.e-12) {
+                  cout << " " << i << " " << offset(me()) + j << " "
+                       << setw(6)
+                       << dat << endl;
+                }
+            }
+        }
+      cout.flush();
+      msg_->sync();
+    }
+  // end debug code
+#endif
+}
+
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memmsg.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memmsg.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memmsg.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,61 @@
+//
+// memmsg.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_memmsg_h
+#define _util_group_memmsg_h
+
+#include <util/group/message.h>
+#include <util/group/memory.h>
+
+namespace sc {
+
+/** A MsgMemoryGrp that initializes its data using a messagegrp. */
+class MsgMemoryGrp: public MemoryGrp {
+
+  protected:
+    Ref<MessageGrp> msg_;
+  public:
+    MsgMemoryGrp(const Ref<MessageGrp>& msg);
+    MsgMemoryGrp(const Ref<KeyVal>& keyval);
+    ~MsgMemoryGrp();
+    void set_localsize(size_t localsize);
+
+    void sync();
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memmtmpi.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memmtmpi.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memmtmpi.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,452 @@
+//
+// memmtmpi.cc
+// based on memmpi.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_memmtmpi_cc
+#define _util_group_memmtmpi_cc
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#include <assert.h>
+
+#include <util/misc/formio.h>
+#include <util/group/messmpi.h>
+#include <util/group/memmtmpi.h>
+
+#define MPICH_SKIP_MPICXX
+#include <mpi.h>
+
+using namespace std;
+
+// Define this to use immediate mode.  This was added added to work
+// around bugs in non-immediate mode optimizations in an MPI impl.
+#undef USE_IMMEDIATE_MODE
+
+namespace sc {
+
+static const int dbufsize = 32768;
+
+///////////////////////////////////////////////////////////////////////
+// The MTMPIThread class
+
+class MTMPIThread: public Thread {
+  private:
+    MTMPIMemoryGrp *mem_;
+    int req_tag_;
+    int tag_;
+    unsigned int nreq_recd_;
+    double chunk[dbufsize];
+  public:
+    MTMPIThread(MTMPIMemoryGrp *, int reqtype, int tag);
+    void run();
+    int run_one();
+    unsigned int nreq_recd() { return nreq_recd_; }
+    void set_nreq_recd(unsigned int val) { nreq_recd_ = val; }
+};
+
+MTMPIThread::MTMPIThread(MTMPIMemoryGrp *mem,
+                         int reqtype, int tag)
+{
+  mem_ = mem;
+  req_tag_ = reqtype;
+  tag_ = tag;
+  nreq_recd_ = 0;
+}
+
+void
+MTMPIThread::run()
+{
+  while(run_one());
+}
+
+int
+MTMPIThread::run_one()
+{
+  int i;
+  int dsize;
+  int dremain;
+  int doffset;
+  long l;
+  MemoryDataRequest req;
+  MPI_Status status;
+#ifndef USE_IMMEDIATE_MODE
+  MPI_Recv(req.data(),req.nbytes(),MPI_BYTE,MPI_ANY_SOURCE,
+           req_tag_,mem_->comm_comm_,&status);
+#else
+  MPI_Request mpireq;
+  MPI_Irecv(req.data(),req.nbytes(),MPI_BYTE,MPI_ANY_SOURCE,
+           req_tag_,mem_->comm_comm_,&mpireq);
+  MPI_Wait(&mpireq,&status);
+#endif // USE_IMMEDIATE_MODE
+  int rtag = req.serial_number();
+  if (mem_->debug_) {
+      mem_->print_lock_->lock();
+      req.print("RECV",mem_->hout);
+      mem_->print_lock_->unlock();
+    }
+  if (req.touches_data()) {
+      assert(req.size() >= 0);
+      if (req.offset()+req.size() > mem_->localsize()) {
+          mem_->print_lock_->lock();
+          req.print("BAD RECV");
+          ExEnv::outn() << "mem_->localsize() = " << mem_->localsize() << endl;
+          mem_->print_lock_->lock();
+        }
+      assert(req.offset()+req.size() <= mem_->localsize());
+    }
+  switch (req.request()) {
+  case MemoryDataRequest::Deactivate:
+      return 0;
+  case MemoryDataRequest::Retrieve:
+      nreq_recd_++;
+      if (req.lock())
+          mem_->obtain_local_lock(req.offset(), req.offset()+req.size());
+      MPI_Send(&mem_->data_[req.offset()],req.size(),MPI_BYTE,
+               req.node(),rtag,mem_->comp_comm_);
+      break;
+  case MemoryDataRequest::Replace:
+      nreq_recd_++;
+      // May be able to get rid of this MPI_Send - MLL
+      MPI_Send(&tag_,1,MPI_INT,req.node(),rtag,mem_->comp_comm_);
+      MPI_Recv(&mem_->data_[req.offset()],req.size(),MPI_BYTE,
+               req.node(),tag_,mem_->comm_comm_,&status);
+      if (req.lock())
+          mem_->release_local_lock(req.offset(), req.offset()+req.size());
+      break;
+  case MemoryDataRequest::DoubleSum:
+      nreq_recd_++;
+//      MPI_Send(&tag_,1,MPI_INT,req.node(),rtag,mem_->comm_);
+      dsize = req.size()/sizeof(double);
+      dremain = dsize;
+      doffset = req.offset()/sizeof(double);
+      mem_->obtain_local_lock(req.offset(), req.offset()+req.size());
+      while(dremain>0) {
+          int dchunksize = dbufsize;
+          if (dremain < dchunksize) dchunksize = dremain;
+#ifndef USE_IMMEDIATE_MODE
+          MPI_Recv(chunk,dchunksize,MPI_DOUBLE,
+                   req.node(),rtag ,mem_->comm_comm_,&status);
+#else
+          MPI_Request mpireq;
+          MPI_Irecv(chunk,dchunksize,MPI_DOUBLE,
+                   req.node(),rtag ,mem_->comm_comm_,&mpireq);
+          MPI_Wait(&mpireq,&status);
+#endif // USE_IMMEDIATE_MODE
+          double *source_data = &((double*)mem_->data_)[doffset];
+          for (i=0; i<dchunksize; i++) {
+              source_data[i] += chunk[i];
+            }
+          dremain -= dchunksize;
+          doffset += dchunksize;
+        }
+      mem_->release_local_lock(req.offset(), req.offset()+req.size());
+      break;
+  default:
+      mem_->print_lock_->lock();
+      ExEnv::outn() << "MTMPIThread: bad memory data request" << endl;
+      mem_->print_lock_->unlock();
+      abort();
+    }
+  return 1;
+}
+
+///////////////////////////////////////////////////////////////////////
+// The MTMPIMemoryGrp class
+
+static ClassDesc MTMPIMemoryGrp_cd(
+  typeid(MTMPIMemoryGrp),"MTMPIMemoryGrp",1,"public ActiveMsgMemoryGrp",
+  0, create<MTMPIMemoryGrp>, 0);
+
+MTMPIMemoryGrp::MTMPIMemoryGrp(const Ref<MessageGrp>& msg,
+                               const Ref<ThreadGrp>& th,
+                               MPI_Comm comm):
+  ActiveMsgMemoryGrp(msg)
+{
+  if (debug_) ExEnv::outn() << "MTMPIMemoryGrp CTOR entered" << endl;
+
+  th_ = th;
+
+  init_mtmpimg(comm,th_->nthread());
+}
+
+MTMPIMemoryGrp::MTMPIMemoryGrp(const Ref<KeyVal>& keyval):
+  ActiveMsgMemoryGrp(keyval)
+{
+  if (debug_) ExEnv::outn() << "MTMPIMemoryGrp keyval CTOR entered" << endl;
+
+  th_ = ThreadGrp::get_default_threadgrp();
+
+  KeyValValueint nthreaddef(th_->nthread());
+  int nthread = keyval->intvalue("num_threads",nthreaddef);
+  ExEnv::out0() << indent << "MTMPIMemoryGrp: num_threads = " << nthread << endl;
+
+  init_mtmpimg(MPI_COMM_WORLD,nthread);
+}
+
+MTMPIMemoryGrp::~MTMPIMemoryGrp()
+{
+  deactivate();
+  for (int i=0; i<th_->nthread()-1; i++) {
+      delete thread_[i];
+    }
+  delete[] thread_;
+  delete[] nreq_sent_;
+  delete[] nreq_sent_buf_;
+}
+
+void
+MTMPIMemoryGrp::init_mtmpimg(MPI_Comm comm, int nthread)
+{
+  int i;
+  active_ = 0;
+
+  if (nthread < 2) nthread = 2;
+  th_ = th_->clone(nthread);
+  nthread = th_->nthread();
+
+  if (debug_) {
+      char name[256];
+      sprintf(name, "mpqc.hand.%d", me());
+      hout.open(name);
+      sprintf(name, "mpqc.main.%d", me());
+      mout.open(name);
+    }
+
+  MPI_Comm_dup(comm, &comp_comm_);
+  MPI_Comm_dup(comm, &comm_comm_);
+
+  MPI_Errhandler_set(comp_comm_, MPI_ERRORS_ARE_FATAL);
+  MPI_Errhandler_set(comm_comm_, MPI_ERRORS_ARE_FATAL);
+
+  serial_ = 0;
+  req_tag_ = 15001;
+
+  serial_lock_ = th_->new_lock();
+
+  thread_ = new MTMPIThread*[nthread-1];
+  th_->add_thread(0,0);
+  for (i=1; i<nthread; i++) {
+      thread_[i-1] = new MTMPIThread(this,req_tag_,req_tag_ + i);
+      th_->add_thread(i,thread_[i-1]);
+    }
+  print_lock_ = th_->new_lock();
+
+  nreq_sent_ = new unsigned int[n()];
+  memset(nreq_sent_, 0, sizeof(unsigned int)*n());
+  nreq_sent_buf_ = new unsigned int[n()];
+}
+
+int
+MTMPIMemoryGrp::serial(int node)
+{
+  serial_lock_->lock();
+  nreq_sent_[node]++;
+  int r = serial_;
+  serial_++;
+  if (serial_ == req_tag_) serial_ = 0;
+  serial_lock_->unlock();
+  return r;
+}
+
+void
+MTMPIMemoryGrp::retrieve_data(void *data, int node, int offset, int size,
+                              int lock)
+{
+  MemoryDataRequest req(MemoryDataRequest::Retrieve,me(),offset,size,lock,
+                        serial(node));
+  int tag = req.serial_number();
+
+  // send the request
+  if (debug_) {
+      print_lock_->lock();
+      req.print("SEND",mout);
+      print_lock_->unlock();
+    }
+  MPI_Send(req.data(),req.nbytes(),MPI_BYTE,node,req_tag_,comm_comm_);
+
+  // receive the data
+  MPI_Status status;
+  MPI_Recv(data,size,MPI_BYTE,node,tag,comp_comm_,&status);
+}
+
+void
+MTMPIMemoryGrp::replace_data(void *data, int node, int offset, int size,
+                             int unlock)
+{
+  MemoryDataRequest req(MemoryDataRequest::Replace,me(),offset,size,unlock,
+                        serial(node));
+  int tag = req.serial_number();
+
+  if (debug_) {
+      print_lock_->lock();
+      req.print("SEND",mout);
+      print_lock_->unlock();
+    }
+  MPI_Send(req.data(),req.nbytes(),MPI_BYTE,node,req_tag_,comm_comm_);
+
+  // wait for the go ahead message
+  MPI_Status status;
+  int rtag;
+  MPI_Recv(&rtag,1,MPI_INT,node,tag,comp_comm_,&status);
+
+  // send the data
+  MPI_Send(data,size,MPI_BYTE,node,rtag,comm_comm_);
+}
+
+void
+MTMPIMemoryGrp::sum_data(double *data, int node, int offset, int size)
+{
+  MemoryDataRequest req(MemoryDataRequest::DoubleSum,me(),offset,size,
+                        0, serial(node));
+  int tag = req.serial_number();
+
+  // send the request
+  if (debug_) {
+      print_lock_->lock();
+      req.print("SEND",mout);
+      print_lock_->unlock();
+    }
+#ifndef USE_IMMEDIATE_MODE
+  MPI_Send(req.data(),req.nbytes(),MPI_BYTE,node,req_tag_,comm_comm_);
+#else
+  MPI_Status status;
+  MPI_Request mpireq;
+  MPI_Isend(req.data(),req.nbytes(),MPI_BYTE,node,req_tag_,comm_comm_,&mpireq);
+  MPI_Wait(&mpireq,&status);
+#endif // USE_IMMEDIATE_MODE
+
+  // wait for the go ahead message
+//  MPI_Status status;
+//  int rtag;
+//  MPI_Recv(&rtag,1,MPI_INT,node,tag,comm_,&status);
+
+  int dsize = size/sizeof(double);
+  int dremain = dsize;
+  int dcurrent = 0;
+  while(dremain>0) {
+      int dchunksize = dbufsize;
+      if (dremain < dchunksize) dchunksize = dremain;
+      // send the data
+#ifndef USE_IMMEDIATE_MODE
+      MPI_Send(&data[dcurrent],dchunksize,MPI_DOUBLE,
+               node,tag,comm_comm_);
+#else
+      MPI_Request mpireq;
+      MPI_Isend(&data[dcurrent],dchunksize,MPI_DOUBLE,
+               node,tag,comm_comm_,&mpireq);
+      MPI_Wait(&mpireq,&status);
+#endif // USE_IMMEDIATE_MODE
+      dcurrent += dchunksize;
+      dremain -= dchunksize;
+    }
+}
+
+void
+MTMPIMemoryGrp::activate()
+{
+  // Only remote requests require the handler.  There are only remote
+  // requests if there is more than one node.
+  if (n() == 1) return;
+
+  if (th_->nthread() < 2) {
+      ExEnv::outn() << "MTMPIMemoryGrp didn't get enough threads" << endl;
+      abort();
+    }
+
+  if (active_) return;
+  active_ = 1;
+
+  th_->start_threads();
+}
+
+void
+MTMPIMemoryGrp::deactivate()
+{
+  if (!active_) return;
+  active_ = 0;
+
+  // send a shutdown message
+  MemoryDataRequest req(MemoryDataRequest::Deactivate);
+  if (debug_) {
+      print_lock_->lock();
+      req.print("SEND",mout);
+      print_lock_->unlock();
+    }
+  for (int i=1; i<th_->nthread(); i++) {
+#ifndef USE_IMMEDIATE_MODE
+      MPI_Send(req.data(),req.nbytes(),MPI_BYTE,me(),req_tag_,comm_comm_);
+#else
+      MPI_Request mpireq;
+      MPI_Status status;
+      MPI_Isend(req.data(),req.nbytes(),MPI_BYTE,me(),req_tag_,comm_comm_,&mpireq);
+      MPI_Wait(&mpireq,&status);
+#endif // USE_IMMEDIATE_MODE
+    }
+
+  // wait on the thread to shutdown
+  th_->wait_threads();
+}
+
+void
+MTMPIMemoryGrp::sync()
+{
+  if (active_) {
+      MPI_Allreduce(nreq_sent_, nreq_sent_buf_,
+                    n(), MPI_UNSIGNED, MPI_SUM, comm_comm_);
+      deactivate();
+      unsigned int nreq_recd = 0;
+      for (int i=0; i<th_->nthread()-1; i++) {
+          nreq_recd += thread_[i]->nreq_recd();
+          thread_[i]->set_nreq_recd(0);
+        }
+      int n_outstanding = nreq_sent_buf_[me()] - nreq_recd;
+      for (int i=0; i<n_outstanding; i++) {
+          thread_[0]->run_one();
+        }
+      memset(nreq_sent_, 0, sizeof(unsigned int)*n());
+      // Make sure processing of all outstanding requests is finished
+      // before starting the next phase.
+      MPI_Barrier(comm_comm_);
+      activate();
+    }
+  else {
+      MPI_Barrier(comm_comm_);
+    }
+}
+
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+
+}
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memmtmpi.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memmtmpi.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memmtmpi.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,107 @@
+//
+// memmtmpi.h
+// based on memmpi.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_memmtmpi_h
+#define _util_group_memmtmpi_h
+
+#include <fstream>
+#define MPICH_SKIP_MPICXX
+#include <mpi.h>
+
+#include <util/group/message.h>
+#include <util/group/memamsg.h>
+#include <util/group/thread.h>
+
+namespace sc {
+
+class MTMPIThread;
+
+/** This MemoryGrp class requires a MT-safe MPI implementation.  The
+default MessageGrp must be a MPIMessageGrp.  MPI must be safe with respect
+to the default ThreadGrp.  Alternately, a MessageGrp and a ThreadGrp can be
+passed to the constructor.  */
+class MTMPIMemoryGrp: public ActiveMsgMemoryGrp {
+  private:
+    Ref<ThreadGrp> th_;
+
+    Ref<ThreadLock> serial_lock_;
+    int serial_;
+    int serial(int node);
+
+    MPI_Comm comp_comm_;
+    MPI_Comm comm_comm_;
+    int req_tag_;
+
+    int active_;
+
+    unsigned int *nreq_sent_;
+    unsigned int *nreq_sent_buf_;
+
+    MTMPIThread **thread_;
+    Ref<ThreadLock> print_lock_; // needed for debugging only
+    std::ofstream hout; // handler out
+    std::ofstream mout; // main thread out
+
+    void init_mtmpimg(MPI_Comm comm, int nthreads);
+
+    // parent class pure virtuals
+    void retrieve_data(void *, int node, int offset, int size, int lock);
+    void replace_data(void *, int node, int offset, int size, int unlock);
+    void sum_data(double *data, int node, int doffset, int dsize);
+
+    friend class MTMPIThread;
+  public:
+    /** Construct a MTMPIMemoryGrp given a MessageGrp, ThreadGrp, and
+        an MPI communicator.  The communicator can be a subset of
+        MPI_COMM_WORLD, in which case, the MessageGrp must refer to the
+        same subset. */
+    MTMPIMemoryGrp(const Ref<MessageGrp>& msg, const Ref<ThreadGrp> &th,
+                   MPI_Comm comm = MPI_COMM_WORLD);
+    /** Construct a MTMPIMemoryGrp given a KeyVal input object. A
+        fully thread safe MPI is needed (MPI_THREAD_MULTIPLE). */
+    MTMPIMemoryGrp(const Ref<KeyVal> &);
+    ~MTMPIMemoryGrp();
+
+    void activate();
+    void deactivate();
+
+    void sync();
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memory.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memory.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memory.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,368 @@
+//
+// memory.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#ifdef HAVE_CONFIG_H
+#include <scconfig.h>
+#endif
+
+#include <scconfig.h>
+#include <util/misc/formio.h>
+#include <util/group/memory.h>
+
+#include <util/group/memproc.h>
+
+#ifdef HAVE_SYSV_IPC
+#  include <util/group/messshm.h>
+#  include <util/group/memshm.h>
+#endif
+
+#if defined(HAVE_MPI)
+#  include <util/group/messmpi.h>
+#  include <util/group/memmtmpi.h>
+#endif
+
+#if defined(HAVE_ARMCI)
+#  include <util/group/memarmci.h>
+#endif
+
+using namespace std;
+using namespace sc;
+
+//////////////////////////////////////////////////////////////////////
+// MemoryGrpBuf template instantiations
+
+#ifdef EXPLICIT_TEMPLATE_INSTANTIATION
+template class MemoryGrpBuf<double>;
+template class MemoryGrpBuf<int>;
+template class MemoryGrpBuf<char>;
+template class MemoryGrpBuf<unsigned char>;
+#endif
+
+//////////////////////////////////////////////////////////////////////
+// MemoryGrp members
+
+static ClassDesc MemoryGrp_cd(
+  typeid(MemoryGrp),"MemoryGrp",1,"public DescribedClass",
+  0, 0, 0);
+
+MemoryGrp::MemoryGrp()
+{
+  debug_ = 0;
+
+  offsets_ = 0;
+
+  init_locks();
+}
+
+MemoryGrp::MemoryGrp(const Ref<KeyVal>& keyval)
+{
+  debug_ = keyval->intvalue("debug");
+
+  offsets_ = 0;
+
+  init_locks();
+}
+
+MemoryGrp::~MemoryGrp()
+{
+  delete[] offsets_;
+  delete[] locks_;
+}
+
+void
+MemoryGrp::init_locks()
+{
+  Ref<ThreadGrp> thgrp = ThreadGrp::get_default_threadgrp();
+  nlock_ = 2 * thgrp->nthread();
+  locks_ = new Ref<ThreadLock>[nlock_];
+  for (int i=0; i<nlock_; i++) locks_[i] = thgrp->new_lock();
+}
+
+MemoryGrp *
+MemoryGrp::initial_memorygrp()
+{
+  int argc = 0;
+  return initial_memorygrp(argc,0);
+}
+
+MemoryGrp *
+MemoryGrp::initial_memorygrp(int &argc, char *argv[])
+{
+  MemoryGrp *grp = 0;
+
+  char *keyval_string = 0;
+
+  // see if a memory group is given on the command line
+  if (argc && argv) {
+      for (int i=0; i<argc; i++) {
+	  if (argv[i] && !strcmp(argv[i], "-memorygrp")) {
+              char *memorygrp_string = argv[i];
+              i++;
+              if (i >= argc) {
+                  ExEnv::errn() << "-memorygrp must be following by an argument"
+                       << endl;
+                  abort();
+                }
+              keyval_string = argv[i];
+              // move the memorygrp arguments to the end of argv
+              int j;
+              for (j=i+1; j<argc; j++) {
+                  argv[j-2] = argv[j];
+                }
+              argv[j++] = memorygrp_string;
+              argv[j++] = keyval_string;
+              // decrement argc to hide the last two arguments
+              argc -= 2;
+              break;
+            }
+        }
+    }
+
+  if (!keyval_string) {
+      // find out if the environment gives the containing memory group
+      keyval_string = getenv("MEMORYGRP");
+      if (keyval_string) {
+          if (!strncmp("MEMORYGRP=", keyval_string, 11)) {
+              keyval_string = strchr(keyval_string, '=');
+            }
+          if (*keyval_string == '=') keyval_string++;
+        }
+    }
+
+  // if keyval input for a memory group was found, then
+  // create it.
+  if (keyval_string) {
+      //ExEnv::outn() << "Creating MemoryGrp from \"" << keyval_string << "\"" << endl;
+      Ref<ParsedKeyVal> strkv = new ParsedKeyVal();
+      strkv->parse_string(keyval_string);
+      Ref<DescribedClass> dc = strkv->describedclassvalue();
+      grp = dynamic_cast<MemoryGrp*>(dc.pointer());
+      if (dc.null()) {
+          ExEnv::errn() << "initial_memorygrp: couldn't find a MemoryGrp in "
+               << keyval_string << endl;
+          abort();
+        }
+      else if (!grp) {
+          ExEnv::errn() << "initial_memorygrp: wanted MemoryGrp but got "
+               << dc->class_name() << endl;
+          abort();
+        }
+      // prevent an accidental delete
+      grp->reference();
+      strkv = 0;
+      dc = 0;
+      // accidental delete not a problem anymore since all smart pointers
+      // to grp are dead
+      grp->dereference();
+      return grp;
+    }
+
+  return grp;
+}
+
+void
+MemoryGrp::activate()
+{
+}
+
+void
+MemoryGrp::deactivate()
+{
+}
+
+void
+MemoryGrp::print(ostream&o) const
+{
+  o << scprintf("MemoryGrp (node %d):\n", me());
+  o << scprintf("%d: n = %d\n", me(), n());
+  for (int i=0; i<=n_; i++) {
+      o << scprintf("%d: offset[%d] = %5d\n", me(), i, offsets_[i]);
+    }
+}
+
+void
+MemoryGrp::sum_reduction(double *data, distsize_t doffset, int dlength)
+{
+  distsize_t offset = doffset * sizeof(double);
+  int length = dlength * sizeof(double);
+
+  if (offset + length > totalsize()) {
+      ExEnv::errn() << "MemoryGrp::sum_reduction: arg out of range:"
+                    << " offset = " << double(offset)
+                    << " length = " << length
+                    << " totalsize() = " << double(totalsize())
+                    << endl;
+      abort();
+    }
+
+  double *source_data = (double*) obtain_readwrite(offset, length);
+
+  for (int i=0; i<dlength; i++) {
+      source_data[i] += data[i];
+    }
+
+  release_readwrite((void*) source_data, offset, length);
+}
+
+void
+MemoryGrp::sum_reduction_on_node(double *data, size_t doffset, int dlength,
+                                 int node)
+{
+  if (node == -1) node = me();
+
+  sum_reduction(data, doffset + offset(node)/sizeof(double),
+                dlength);
+}
+
+void*
+MemoryGrp::malloc_local(size_t nbyte)
+{
+  return new char[nbyte];
+}
+
+void
+MemoryGrp::free_local(void *data)
+{
+  delete[] reinterpret_cast<char*>(data);
+}
+
+double*
+MemoryGrp::malloc_local_double(size_t ndouble)
+{
+  return reinterpret_cast<double*>(malloc_local(ndouble*sizeof(double)));
+}
+
+void
+MemoryGrp::free_local_double(double *data)
+{
+  free_local(data);
+}
+
+void
+MemoryGrp::catchup()
+{
+  return;
+}
+
+void
+MemoryGrp::obtain_local_lock(size_t start, size_t fence)
+{
+  distsize_t locked_region_size = 1 + localsize()/nlock_;
+  int lstart = start/locked_region_size;
+  int llast = fence/locked_region_size;
+  for (int i=lstart; i<=llast; i++) {
+      locks_[i]->lock();
+    }
+}
+
+void
+MemoryGrp::release_local_lock(size_t start, size_t fence)
+{
+  distsize_t locked_region_size = 1 + localsize()/nlock_;
+  int lstart = start/locked_region_size;
+  int llast = fence/locked_region_size;
+  for (int i=lstart; i<=llast; i++) {
+      locks_[i]->unlock();
+    }
+}
+
+static Ref<MemoryGrp> default_memorygrp;
+
+void
+MemoryGrp::set_default_memorygrp(const Ref<MemoryGrp>& grp)
+{
+  default_memorygrp = grp;
+}
+
+MemoryGrp*
+MemoryGrp::get_default_memorygrp()
+{
+  if (default_memorygrp.nonnull()) return default_memorygrp.pointer();
+
+  Ref<MessageGrp> msg = MessageGrp::get_default_messagegrp();
+
+#if defined(HAVE_MPI) && defined(DEFAULT_MTMPI)
+  Ref<ThreadGrp> thr = ThreadGrp::get_default_threadgrp();
+  default_memorygrp = new MTMPIMemoryGrp(msg,thr);
+  return default_memorygrp.pointer();
+#endif
+
+#if defined(DEFAULT_ARMCI)
+  default_memorygrp = new ARMCIMemoryGrp(msg);
+  return default_memorygrp.pointer();
+#endif
+
+  if (msg.null()) {
+      ExEnv::errn() << scprintf("MemoryGrp::get_default_memorygrp: requires default MessageGrp if default behavior not configured\n");
+      abort();
+    }
+#if defined(HAVE_MPI)
+  else if (msg->class_desc() == ::class_desc<MPIMessageGrp>()) {
+      Ref<ThreadGrp> thr = ThreadGrp::get_default_threadgrp();
+      default_memorygrp = new MTMPIMemoryGrp(msg,thr);
+      return default_memorygrp.pointer();
+    }
+#endif
+#if defined(HAVE_ARMCI)
+  else if (msg->class_desc() == ::class_desc<MPIMessageGrp>()) {
+      default_memorygrp = new ARMCIMemoryGrp(msg);
+      return default_memorygrp.pointer();
+    }
+#endif
+#ifdef HAVE_SYSV_IPC
+  else if (msg->class_desc() == ::class_desc<ShmMessageGrp>()) {
+      default_memorygrp = new ShmMemoryGrp(msg);
+      return default_memorygrp.pointer();
+    }
+#endif
+  else if (msg->n() == 1) {
+      default_memorygrp = new ProcMemoryGrp();
+      return default_memorygrp.pointer();
+    }
+  else {
+      ExEnv::errn() << scprintf("MemoryGrp::get_default_memorygrp: cannot create "
+              "default for \"%s\"\n.", msg->class_name());
+      abort();
+    }
+
+  if (default_memorygrp.null()) {
+      ExEnv::err0() << scprintf("WARNING: MemoryGrp::get_default_memorygrp(): failed\n");
+      default_memorygrp = new ProcMemoryGrp;
+    }
+  return default_memorygrp.pointer();
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memory.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memory.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memory.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,391 @@
+//
+// memory.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_memory_h
+#define _util_group_memory_h
+
+#include <iostream>
+
+#include <scconfig.h>
+#include <util/class/class.h>
+#include <util/group/thread.h>
+
+namespace sc {
+
+#if 0 // this can be used to catch accidental conversions to int
+class distsize_t {
+    friend size_t distsize_to_size(const distsize_t &a);
+    friend distsize_t operator *(const int &a,const distsize_t &b);
+    friend distsize_t operator +(const int &a,const distsize_t &b);
+    friend distsize_t operator -(const int &a,const distsize_t &b);
+    friend distsize_t operator /(const int &a,const distsize_t &b);
+    friend distsize_t operator %(const int &a,const distsize_t &b);
+    friend ostream& operator <<(ostream& o, const distsize_t &s);
+  private:
+    unsigned long long s;
+  public:
+    distsize_t(): s(999999999999999LL) {}
+    distsize_t(int a): s(a) {}
+    distsize_t(unsigned int a): s(a) {}
+    distsize_t(unsigned long long a): s(a) {}
+    distsize_t &operator =(const distsize_t &a)
+        { s=a.s; return *this; }
+    distsize_t &operator +=(const distsize_t &a)
+        { s+=a.s; return *this; }
+    distsize_t operator *(const distsize_t &a) const
+        { return s*a.s; }
+    distsize_t operator +(const distsize_t &a) const
+        { return s+a.s; }
+    distsize_t operator -(const distsize_t &a) const
+        { return s-a.s; }
+    distsize_t operator /(const distsize_t &a) const
+        { return s/a.s; }
+    distsize_t operator %(const distsize_t &a) const
+        { return s%a.s; }
+    bool operator <(const distsize_t &a) const
+        { return s<a.s; }
+    bool operator <=(const distsize_t &a) const
+        { return s<=a.s; }
+    bool operator >(const distsize_t &a) const
+        { return s>a.s; }
+    bool operator >=(const distsize_t &a) const
+        { return s>=a.s; }
+    bool operator ==(const distsize_t &a) const
+        { return s==a.s; }
+    distsize_t operator *(const int &a) const
+        { return s*a; }
+    distsize_t operator +(const int &a) const
+        { return s+a; }
+    distsize_t operator -(const int &a) const
+        { return s-a; }
+    distsize_t operator /(const int &a) const
+        { return s/a; }
+    distsize_t operator %(const int &a) const
+        { return s%a; }
+};
+inline distsize_t operator *(const int &a,const distsize_t &b)
+{ return a*b.s; }
+inline distsize_t operator +(const int &a,const distsize_t &b)
+{ return a+b.s; }
+inline distsize_t operator -(const int &a,const distsize_t &b)
+{ return a-b.s; }
+inline distsize_t operator /(const int &a,const distsize_t &b)
+{ return a/b.s; }
+inline distsize_t operator %(const int &a,const distsize_t &b)
+{ return a%b.s; }
+inline ostream& operator <<(ostream& o, const distsize_t &s) { return o<<s.s; }
+inline size_t distsize_to_size(const distsize_t &a) {return a.s;}
+#elif defined(HAVE_LONG_LONG)
+typedef unsigned long long distsize_t;
+typedef long long distssize_t;
+inline size_t distsize_to_size(const distsize_t &a) {return a;}
+#else
+typedef unsigned long distsize_t;
+typedef long distssize_t;
+inline size_t distsize_to_size(const distsize_t &a) {return a;}
+#endif
+
+/** The MemoryGrp abstract class provides a way of accessing distributed
+memory in a parallel machine.  Several specializations are available.  For
+one processor, ProcMemoryGrp provides a simple stub implementation.
+Parallel specializations include ShmMemoryGrp, MTMPIMemoryGrp, and
+ARMCIMemoryGrp.  The particular specializations that work depend highly on
+the target hardware and software environment.
+
+*/
+class MemoryGrp: public DescribedClass {
+  private:
+    Ref<ThreadLock> *locks_;
+    int nlock_;
+ 
+    void init_locks();
+
+
+  protected:
+    // derived classes must fill in all these
+    // ~MemoryGrp deletes the arrays
+    int me_;
+    int n_;
+    distsize_t *offsets_; // offsets_[n_] is the fence for all data
+
+    // set to nonzero for debugging information
+    int debug_;
+
+    void obtain_local_lock(size_t start, size_t fence);
+    void release_local_lock(size_t start, size_t fence);
+  public:
+    MemoryGrp();
+    MemoryGrp(const Ref<KeyVal>&);
+    virtual ~MemoryGrp();
+    
+    /// Returns who I am.
+    int me() const { return me_; }
+    /// Returns how many nodes there are.
+    int n() const { return n_; }
+
+    /** Set the size of locally held memory.
+        When memory is accessed using a global offset counting
+        starts at node 0 and proceeds up to node n() - 1. */
+    virtual void set_localsize(size_t) = 0;
+    /// Returns the amount of memory residing locally on me().
+    size_t localsize() { return distsize_to_size(offsets_[me_+1]-offsets_[me_]); }
+    /// Returns a pointer to the local data.
+    virtual void *localdata() = 0;
+    /// Returns the global offset to this node's memory.
+    distsize_t localoffset() { return offsets_[me_]; }
+    /// Returns the amount of memory residing on node.
+    int size(int node)
+        { return distsize_to_size(offsets_[node+1] - offsets_[node]); }
+    /// Returns the global offset to node's memory.
+    distsize_t offset(int node) { return offsets_[node]; }
+    /// Returns the sum of all memory allocated on all nodes.
+    distsize_t totalsize() { return offsets_[n_]; }
+
+    /// Activate is called before the memory is to be used.
+    virtual void activate();
+    /// Deactivate is called after the memory has been used.
+    virtual void deactivate();
+
+    /// This gives write access to the memory location.  No locking is done.
+    virtual void *obtain_writeonly(distsize_t offset, int size) = 0;
+    /** Only one thread can have an unreleased obtain_readwrite at a time.
+        The actual memory region locked can be larger than that requested.
+        If the memory region is already locked this will block.  For this
+        reason, data should be held as read/write for as short a time as
+        possible. */
+    virtual void *obtain_readwrite(distsize_t offset, int size) = 0;
+    /// This gives read access to the memory location.  No locking is done.
+    virtual void *obtain_readonly(distsize_t offset, int size) = 0;
+    /// This is called when read access is no longer needed.
+    virtual void release_readonly(void *data, distsize_t offset, int size) = 0;
+    /// This is called when write access is no longer needed.
+    virtual void release_writeonly(void *data, distsize_t offset, int size)=0;
+    /** This is called when read/write access is no longer needed.
+        The memory will be unlocked. */
+    virtual void release_readwrite(void *data, distsize_t offset, int size)=0;
+
+    virtual void sum_reduction(double *data, distsize_t doffset, int dsize);
+    virtual void sum_reduction_on_node(double *data, size_t doffset, int dsize,
+                                       int node = -1);
+
+    /** Synchronizes all the nodes.  This is useful after remote memory
+        writes to be certain that all of the writes have completed and the
+        data can be accessed locally, for example. */
+    virtual void sync() = 0;
+
+    /** Allocate data that will be accessed locally only.  Using this
+        for data that will be used for global operation can improve
+        efficiency.  Data allocated in this way must be freed with
+        free_local_double.  */
+    virtual void* malloc_local(size_t nbyte);
+    virtual double* malloc_local_double(size_t ndouble);
+
+    /** Free data that was allocated with malloc_local_double. */
+    virtual void free_local(void *data);
+    virtual void free_local_double(double *data);
+
+    /** Processes outstanding requests. Some memory group implementations
+        don't have access to real shared memory or even active messages.
+        Instead, requests are processed whenever certain memory group
+        routines are called.  This can cause large latencies and buffer
+        overflows.  If this is a problem, then the catchup member can be
+        called to process all outstanding requests. */
+    virtual void catchup();
+
+    /// Prints out information about the object.
+    virtual void print(std::ostream &o = ExEnv::out0()) const;
+
+    /** Create a memory group.  This routine looks for a -memorygrp
+        argument, and then the environmental variable MEMORYGRP to decide
+        which specialization of MemoryGrp would be appropriate.  The
+        argument to -memorygrp or the value of the environmental variable
+        should be either string for a ParsedKeyVal constructor or a
+        classname.  The default ThreadGrp and MessageGrp objects should be
+        initialized before this is called. */
+    static MemoryGrp* initial_memorygrp(int &argc, char** argv);
+    static MemoryGrp* initial_memorygrp();
+    /** The default memory group contains the primary memory group to
+        be used by an application. */
+    static void set_default_memorygrp(const Ref<MemoryGrp>&);
+    /** Returns the default memory group.  If the default memory
+        group has not yet been set, then one is created.
+        The particular specialization used is determined by
+        configuration options and which specializations are being
+        used for MessageGrp and ThreadGrp. */
+    static MemoryGrp* get_default_memorygrp();
+};
+
+
+/** The MemoryGrpBuf class provides access to pieces of the
+    global shared memory that have been obtained with MemoryGrp.
+    MemoryGrpBuf is a template class that is parameterized on
+    data_t.  All lengths and offsets of given in terms
+    of sizeof(data_t). */
+template <class data_t>
+class MemoryGrpBuf {
+    Ref<MemoryGrp> grp_;
+    enum AccessType { None, Read, Write, ReadWrite };
+    AccessType accesstype_;
+    data_t *data_;
+    distsize_t offset_;
+    int length_;
+  public:
+    /** Creates a new MemoryGrpBuf given a MemoryGrp
+        reference.  This is a template class parameterized on
+        data_t. */
+    MemoryGrpBuf(const Ref<MemoryGrp> &);
+    /** Request write only access to global memory at the global address
+        offset and with size length.  Writing the same bit of memory twice
+        without an intervening sync of the MemoryGrp will have undefined
+        results. */
+    data_t *writeonly(distsize_t offset, int length);
+    /** Request read write access to global memory at the global address
+        offset and with size length.  This will lock the memory it uses
+        until release is called unless locking has been turned off in the
+        MemoryGrp object. */
+    data_t *readwrite(distsize_t offset, int length);
+    /** Request read only access to global memory at the global address
+        offset and with size length.  Writing to the
+        specified region without an intervening sync of the MemoryGrp
+        will have undefined results. */
+    const data_t *readonly(distsize_t offset, int length);
+    /** These behave like writeonly, readwrite, and readonly, except the
+        offset is local to the node specified by node.  If node = -1, then
+        the local node is used. */
+    data_t *writeonly_on_node(size_t offset, int length, int node = -1);
+    data_t *readwrite_on_node(size_t offset, int length, int node = -1);
+    const data_t *readonly_on_node(size_t offset, int length, int node = -1);
+    /** Release the access to the chunk of global memory that was obtained
+        with writeonly, readwrite, readonly, writeonly_on_node,
+        readwrite_on_node, and readonly_on_node. */
+    void release();
+    /// The length of the current bit of memory.
+    int length() const { return length_; }
+};
+
+//////////////////////////////////////////////////////////////////////
+// MemoryGrpBuf members
+
+template <class data_t>
+MemoryGrpBuf<data_t>::MemoryGrpBuf(const Ref<MemoryGrp> & grp)
+{
+  grp_ = grp;
+  accesstype_ = None;
+}
+
+template <class data_t>
+data_t *
+MemoryGrpBuf<data_t>::writeonly(distsize_t offset, int length)
+{
+  if (accesstype_ != None) release();
+  data_ = (data_t *) grp_->obtain_writeonly(sizeof(data_t)*offset,
+                                            sizeof(data_t)*length);
+  offset_ = offset;
+  length_ = length;
+  accesstype_ = Write;
+  return data_;
+}
+
+template <class data_t>
+data_t *
+MemoryGrpBuf<data_t>::readwrite(distsize_t offset, int length)
+{
+  if (accesstype_ != None) release();
+  data_ = (data_t *) grp_->obtain_readwrite(sizeof(data_t)*offset,
+                                            sizeof(data_t)*length);
+  offset_ = offset;
+  length_ = length;
+  accesstype_ = ReadWrite;
+  return data_;
+}
+
+template <class data_t>
+const data_t *
+MemoryGrpBuf<data_t>::readonly(distsize_t offset, int length)
+{
+  if (accesstype_ != None) release();
+  data_ = (data_t *) grp_->obtain_readonly(sizeof(data_t)*offset,
+                                           sizeof(data_t)*length);
+  offset_ = offset;
+  length_ = length;
+  accesstype_ = Read;
+  return data_;
+}
+
+template <class data_t>
+data_t *
+MemoryGrpBuf<data_t>::writeonly_on_node(size_t offset, int length, int node)
+{
+  if (node == -1) node = grp_->me();
+  return writeonly(offset + grp_->offset(node)/sizeof(data_t), length);
+}
+
+template <class data_t>
+data_t *
+MemoryGrpBuf<data_t>::readwrite_on_node(size_t offset, int length, int node)
+{
+  if (node == -1) node = grp_->me();
+  return readwrite(offset + grp_->offset(node)/sizeof(data_t), length);
+}
+
+template <class data_t>
+const data_t *
+MemoryGrpBuf<data_t>::readonly_on_node(size_t offset, int length, int node)
+{
+  if (node == -1) node = grp_->me();
+  return readonly(offset + grp_->offset(node)/sizeof(data_t), length);
+}
+
+template <class data_t>
+void
+MemoryGrpBuf<data_t>::release()
+{
+  if (accesstype_ == Write)
+      grp_->release_writeonly((data_t *)data_,
+                              sizeof(data_t)*offset_, sizeof(data_t)*length_);
+  if (accesstype_ == Read)
+      grp_->release_readonly(data_, sizeof(data_t)*offset_,
+                             sizeof(data_t)*length_);
+  if (accesstype_ == ReadWrite)
+      grp_->release_readwrite(data_, sizeof(data_t)*offset_,
+                              sizeof(data_t)*length_);
+
+  accesstype_ = None;
+}
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memproc.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memproc.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memproc.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,127 @@
+//
+// memproc.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_memproc_cc
+#define _util_group_memproc_cc
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#include <util/group/memproc.h>
+
+using namespace sc;
+
+static ClassDesc ProcMemoryGrp_cd(
+  typeid(ProcMemoryGrp),"ProcMemoryGrp",1,"public MemoryGrp",
+  0, create<ProcMemoryGrp>, 0);
+
+ProcMemoryGrp::ProcMemoryGrp()
+{
+  data_ = 0;
+  offsets_ = 0;
+}
+
+ProcMemoryGrp::ProcMemoryGrp(const Ref<KeyVal>& keyval):
+  MemoryGrp(keyval)
+{
+  data_ = 0;
+  offsets_ = 0;
+}
+
+ProcMemoryGrp::~ProcMemoryGrp()
+{
+  delete[] data_;
+}
+
+void
+ProcMemoryGrp::set_localsize(size_t localsize)
+{
+  delete[] offsets_;
+  delete[] data_;
+  offsets_ = new distsize_t[2];
+  offsets_[0] = 0;
+  offsets_[1] = localsize;
+  n_ = 1;
+  me_ = 0;
+  data_ = new char[localsize];
+}
+
+void *
+ProcMemoryGrp::localdata()
+{
+  return data_;
+}
+
+void *
+ProcMemoryGrp::obtain_readwrite(distsize_t offset, int size)
+{
+  obtain_local_lock(offset-localoffset(), offset-localoffset()+size);
+  return &data_[distsize_to_size(offset)];
+}
+
+void *
+ProcMemoryGrp::obtain_readonly(distsize_t offset, int size)
+{
+  return &data_[distsize_to_size(offset)];
+}
+
+void *
+ProcMemoryGrp::obtain_writeonly(distsize_t offset, int size)
+{
+  return &data_[distsize_to_size(offset)];
+}
+
+void
+ProcMemoryGrp::release_readonly(void *data, distsize_t offset, int size)
+{
+}
+
+void
+ProcMemoryGrp::release_writeonly(void *data, distsize_t offset, int size)
+{
+}
+
+void
+ProcMemoryGrp::release_readwrite(void *data, distsize_t offset, int size)
+{
+  release_local_lock(offset-localoffset(), offset-localoffset()+size);
+}
+
+void
+ProcMemoryGrp::sync()
+{
+}
+
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memproc.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memproc.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memproc.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,71 @@
+//
+// memproc.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_memproc_h
+#define _util_group_memproc_h
+
+#include <sys/types.h>
+
+#include <util/group/memmsg.h>
+
+namespace sc {
+
+/** The ProcMemoryGrp concrete class provides an implementation of
+MemoryGrp for a single processor. */
+class ProcMemoryGrp: public MemoryGrp {
+  private:
+    char *data_;
+  public:
+    ProcMemoryGrp();
+    ProcMemoryGrp(const Ref<KeyVal>&);
+    ~ProcMemoryGrp();
+
+    void set_localsize(size_t);
+    void *localdata();
+
+    void *obtain_readwrite(distsize_t offset, int size);
+    void *obtain_readonly(distsize_t offset, int size);
+    void *obtain_writeonly(distsize_t offset, int size);
+    void release_readonly(void *data, distsize_t offset, int size);
+    void release_writeonly(void *data, distsize_t offset, int size);
+    void release_readwrite(void *data, distsize_t offset, int size);
+
+    void sync();
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memrdma.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memrdma.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memrdma.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,248 @@
+//
+// memrdma.cc
+// Based on memamsg.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_memrdma_cc
+#define _util_group_memrdma_cc
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#include <util/class/scexception.h>
+#include <util/misc/formio.h>
+#include <util/group/pool.h>
+#include <util/group/memrdma.h>
+#include <util/group/memiter.h>
+#include <stdexcept>
+
+using namespace std;
+using namespace sc;
+
+#ifdef HAVE_HRECV
+#  define DISABLE do { masktrap(1); ExEnv::outn().flush(); } while(0)
+#  define ENABLE do { ExEnv::outn().flush(); masktrap(0); } while(0)
+   extern "C" {
+       long masktrap(long state);
+     }
+#else
+#  define DISABLE ExEnv::outn().flush()
+#  define ENABLE ExEnv::outn().flush()
+#endif
+
+#define PRINTF(args) do { DISABLE; \
+                          ExEnv::outn() << scprintf args ; \
+                          ExEnv::outn().flush(); \
+                          ENABLE; \
+                         } while(0)
+
+#undef PRINTF
+#define PRINTF(args)
+
+///////////////////////////////////////////////////////////////////////
+// Members for RDMAMemoryGrp
+
+static ClassDesc RDMAMemoryGrp_cd(
+  typeid(RDMAMemoryGrp),"RDMAMemoryGrp",1,"public MsgMemoryGrp",
+  0, 0, 0);
+
+RDMAMemoryGrp::RDMAMemoryGrp(const Ref<MessageGrp>& msg):
+  MsgMemoryGrp(msg)
+{
+  data_ = 0;
+  default_pool_size_ = 1000000;
+}
+
+RDMAMemoryGrp::RDMAMemoryGrp(const Ref<KeyVal>& keyval):
+  MsgMemoryGrp(keyval)
+{
+  data_ = 0;
+  default_pool_size_ = 1000000;
+}
+
+void*
+RDMAMemoryGrp::malloc_region(size_t nbyte)
+{
+  void *data = 0;
+  for (int i=0; data==0 && i<pools_.size(); i++) {
+      data = pools_[i]->allocate(nbyte);
+    }
+  if (data == 0) {
+      if (default_pool_size_ < nbyte) default_pool_size_ = nbyte * 2;
+      else if (pools_.size() > 4) default_pool_size_ *= 2;
+      void *pooldata = malloc_local(default_pool_size_);
+      Pool *pool = new(pooldata) Pool(default_pool_size_);
+      pools_.push_back(pool);
+      data = pool->allocate(nbyte);
+    }
+  return data;
+}
+
+void
+RDMAMemoryGrp::free_region(void*data)
+{
+  char *cdata = reinterpret_cast<char*>(data);
+  for (int i=0; i<pools_.size(); i++) {
+      char *pstart = reinterpret_cast<char*>(pools_[i]);
+      if (cdata > pstart && cdata < &pstart[pools_[i]->size()]) {
+          pools_[i]->release(data);
+          return;
+        }
+    }
+  throw ProgrammingError("could not find data to release in a Pool",
+                         __FILE__, __LINE__, this->class_desc());
+}
+
+void
+RDMAMemoryGrp::set_localsize(size_t localsize)
+{
+  for (int i=0; i<pools_.size(); i++) {
+      free_local(pools_[i]);
+    }
+  pools_.resize(0);
+
+  MsgMemoryGrp::set_localsize(localsize);
+}
+
+void *
+RDMAMemoryGrp::localdata()
+{
+  return data_;
+}
+
+RDMAMemoryGrp::~RDMAMemoryGrp()
+{
+  deactivate();
+  delete[] data_;
+}
+
+void *
+RDMAMemoryGrp::obtain_writeonly(distsize_t offset, int size)
+{
+  void *data = malloc_region(size);
+  return data;
+}
+
+void *
+RDMAMemoryGrp::obtain_readwrite(distsize_t offset, int size)
+{
+  PRINTF(("RDMAMemoryGrp::obtain_readwrite entered\n"));
+  void *data = malloc_region(size);
+  MemoryIter i(data, offsets_, n());
+  for (i.begin(offset, size); i.ready(); i.next()) {
+      PRINTF(("RDMAMemoryGrp::obtain_readwrite: node = %d, "
+              "int offset = %d, int size = %d\n",
+              i.node(), i.offset()/sizeof(int), i.size()/sizeof(int)));
+      retrieve_data(i.data(), i.node(), i.offset(), i.size(), 1);
+    }
+  PRINTF(("RDMAMemoryGrp::obtain_readwrite exiting\n"));
+  return data;
+}
+
+void *
+RDMAMemoryGrp::obtain_readonly(distsize_t offset, int size)
+{
+  void *data = malloc_region(size);
+  PRINTF(("%d: RDMAMemoryGrp::obtain_readonly:"
+          "overall: offset = %d size = %d\n",
+          me(), offset, size));
+  MemoryIter i(data, offsets_, n());
+  for (i.begin(offset, size); i.ready(); i.next()) {
+      PRINTF(("%d: RDMAMemoryGrp::obtain_readonly:working on:"
+              "node = %d offset = %d size = %d\n",
+              me(), i.node(), i.offset(), i.size()));
+      PRINTF(("RDMAMemoryGrp::obtain_readonly: node = %d, "
+              "int offset = %d, int size = %d\n",
+              i.node(), i.offset()/sizeof(int), i.size()/sizeof(int)));
+      retrieve_data(i.data(), i.node(), i.offset(), i.size(), 0);
+    }
+  return data;
+}
+
+void
+RDMAMemoryGrp::sum_reduction(double *data, distsize_t doffset, int dsize)
+{
+  distsize_t offset = doffset * sizeof(double);
+  int size = dsize * sizeof(double);
+  MemoryIter i(data, offsets_, n());
+  for (i.begin(offset, size); i.ready(); i.next()) {
+      sum_data((double*)i.data(), i.node(), i.offset(), i.size());
+    }
+}
+
+void
+RDMAMemoryGrp::sum_reduction_on_node(double *data, size_t doffset,
+                                          int dlength, int node)
+{
+  if (node == -1) node = me();
+
+  sum_data(data, node, sizeof(double)*doffset, sizeof(double)*dlength);
+}
+
+void
+RDMAMemoryGrp::release_readonly(void *data, distsize_t offset, int size)
+{
+  free_region(data);
+}
+
+void
+RDMAMemoryGrp::release_writeonly(void *data, distsize_t offset, int size)
+{
+  MemoryIter i(data, offsets_, n());
+  for (i.begin(offset, size); i.ready(); i.next()) {
+      PRINTF(("RDMAMemoryGrp::release_write: node = %d, "
+              "int offset = %d, int size = %d\n",
+              i.node(), i.offset()/sizeof(int), i.size()/sizeof(int)));
+      replace_data(i.data(), i.node(), i.offset(), i.size(), 0);
+    }
+  free_region(data);
+}
+
+void
+RDMAMemoryGrp::release_readwrite(void *data, distsize_t offset, int size)
+{
+  MemoryIter i(data, offsets_, n());
+  for (i.begin(offset, size); i.ready(); i.next()) {
+      replace_data(i.data(), i.node(), i.offset(), i.size(), 1);
+    }
+  free_region(data);
+}
+
+void
+RDMAMemoryGrp::print(ostream &o) const
+{
+  MemoryGrp::print(o);
+}
+
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memrdma.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memrdma.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memrdma.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,90 @@
+//
+// memrdma.h
+// Based on memamsg.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_memrdma_h
+#define _util_group_memrdma_h
+
+#include <iostream>
+#include <vector>
+
+#include <util/group/pool.h>
+#include <util/group/memmsg.h>
+
+namespace sc {
+
+/** The RDMAMemoryGrp abstract class specializes the MsgMemoryGrp
+class.  It uses RDMA to implement global shared memory.  */
+class RDMAMemoryGrp : public MsgMemoryGrp {
+  protected:
+    char *data_;
+
+    virtual void retrieve_data(void *, int node, int offset, int size,
+                               int lock) = 0;
+    virtual void replace_data(void *, int node, int offset, int size,
+                              int unlock) = 0;
+    virtual void sum_data(double *data, int node, int doffset, int dsize) = 0;
+
+    std::vector<Pool*> pools_;
+    size_t default_pool_size_;
+    void* malloc_region(size_t nbyte);
+    void free_region(void*);
+  public:
+    RDMAMemoryGrp(const Ref<MessageGrp>& msg);
+    RDMAMemoryGrp(const Ref<KeyVal>&);
+    ~RDMAMemoryGrp();
+
+    void *localdata();
+
+    void set_localsize(size_t localsize);
+
+    void *obtain_writeonly(distsize_t offset, int size);
+    void *obtain_readwrite(distsize_t offset, int size);
+    void *obtain_readonly(distsize_t offset, int size);
+    void release_readonly(void *data, distsize_t offset, int size);
+    void release_writeonly(void *data, distsize_t offset, int size);
+    void release_readwrite(void *data, distsize_t offset, int size);
+
+    void sum_reduction(double *data, distsize_t doffset, int dsize);
+    void sum_reduction_on_node(double *data, size_t doffset, int dsize,
+                               int node = -1);
+
+    void print(std::ostream &o = ExEnv::out0()) const;
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memshm.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memshm.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memshm.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,531 @@
+//
+// memshm.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_memshm_cc
+#define _util_group_memshm_cc
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#include <util/misc/formio.h>
+#include <util/group/pool.h>
+#include <util/group/memshm.h>
+#include <limits.h>
+#include <errno.h>
+
+using namespace std;
+using namespace sc;
+
+#ifndef SHMMAX
+// glibc 2.0.3 isn't defining SHMMAX so make set it here
+#  ifdef __linux__
+#    define SHMMAX 0x1000000
+#  else
+#    define SHMMAX INT_MAX
+#  endif
+#endif
+
+#ifndef SHMMIN
+#define SHMMIN 1
+#endif
+
+#ifndef SIMPLE_LOCK
+#define SIMPLE_LOCK 1
+#endif
+
+#undef DEBUG
+
+static ClassDesc ShmMemoryGrp_cd(
+  typeid(ShmMemoryGrp),"ShmMemoryGrp",1,"public MsgMemoryGrp",
+  0, create<ShmMemoryGrp>, 0);
+
+ShmMemoryGrp::ShmMemoryGrp(const Ref<MessageGrp>& msg):
+  MsgMemoryGrp(msg)
+{
+  update_ = 0;
+  data_ = 0;
+  memory_ = 0;
+  pool_ = 0;
+  rangelock_ = 0;
+  nregion_ = 0;
+  shmid_ = 0;
+  attach_address_ = 0;
+}
+
+ShmMemoryGrp::ShmMemoryGrp(const Ref<KeyVal>& keyval):
+  MsgMemoryGrp(keyval)
+{
+  update_ = 0;
+  data_ = 0;
+  memory_ = 0;
+  pool_ = 0;
+  rangelock_ = 0;
+  nregion_ = 0;
+  shmid_ = 0;
+  attach_address_ = 0;
+}
+
+int
+ShmMemoryGrp::attach_memory(void *ataddress, int size)
+{
+  int i;
+  int fail = 0;
+  int isize;
+  int rsize = size;
+  for (i=0; i<nregion_; i++) {
+      attach_address_[i] = 0;
+    }
+  for (i=0; rsize>0; i++,rsize-=isize) {
+      isize = rsize;
+      if (isize > SHMMAX) isize = SHMMAX;
+      else if (isize < SHMMIN) isize = SHMMIN;
+      if (debug_) {
+          ExEnv::outn() << me() << ": ";
+          ExEnv::outn() << "ShmMemoryGrp: attaching segment with "
+               << isize << " bytes at address " << (void*)ataddress
+               << " on node " << me()
+               << endl;
+        }
+      attach_address_[i] = shmat(shmid_[i],(SHMTYPE)ataddress,0);
+      if (debug_) {
+          ExEnv::outn() << me() << ": ";
+          ExEnv::outn() << "ShmMemoryGrp: got address "
+               << (void*)attach_address_[i]
+               << " on node " << me()
+               << endl;
+        }
+      if (attach_address_[i] == 0
+          || attach_address_[i] == (void*) -1
+          || ataddress && attach_address_[i] != ataddress) {
+          //ExEnv::outn() << "ShmMemoryGrp: shmat: problem attaching using address: "
+          //     << " " << (void*) ataddress
+          //     << ": got address: "
+          //     << (void*) attach_address_[i]
+          //     << " on node " << me()
+          //     << endl;
+          fail = 1;
+        }
+      ataddress = (void*)((char*)(attach_address_[i]) + isize);
+    }
+
+  memory_ = (void*) attach_address_[0];
+
+  if (fail) detach_memory();
+
+  return fail;
+}
+
+void
+ShmMemoryGrp::detach_memory()
+{
+  int i;
+  for (i=0; i<nregion_; i++) {
+      if (attach_address_[i] != 0 && attach_address_[i] != (void*) -1) {
+          if (debug_) {
+              ExEnv::outn() << "detaching " << (void*)attach_address_[i]
+                   << " on node " << me() << endl;
+            }
+          shmdt((SHMTYPE)attach_address_[i]);
+        }
+      attach_address_[i] = 0;
+    }
+}
+
+void
+ShmMemoryGrp::set_localsize(size_t localsize)
+{
+  int i;
+
+  cleanup();
+
+  MsgMemoryGrp::set_localsize(localsize);
+
+  const int poolallocation = 160000;
+
+  update_ = new GlobalCounter[n()];
+
+  // allocate memory both the data and the Pool
+  int size = poolallocation + distsize_to_size(totalsize());
+  // compute the number of shared memory regions that will be needed
+  nregion_ = size/SHMMAX;
+  if (size%SHMMAX) nregion_++;
+  shmid_ = new int[nregion_];
+  attach_address_ = new void*[nregion_];
+
+  // get the shared memory segments
+  if (me() == 0) {
+      int rsize = size;
+      int isize;
+      for (i=0; rsize>0; i++,rsize-=isize) {
+          isize = rsize;
+          if (isize > SHMMAX) isize = SHMMAX;
+          else if (isize < SHMMIN) isize = SHMMIN;
+          if (debug_) {
+              ExEnv::outn() << me() << ": ";
+              ExEnv::outn() << "ShmMemoryGrp: getting segment with " << isize
+                   << " bytes" << endl;
+            }
+          shmid_[i] = shmget(IPC_PRIVATE, isize, IPC_CREAT | SHM_R | SHM_W);
+          if (shmid_[i] == -1) {
+              ExEnv::outn() << me() << ": ";
+              ExEnv::outn() << "ShmMemoryGrp: shmget failed for "
+                   << isize << " bytes: "
+                   << strerror(errno) << endl;
+              abort();
+            }
+        }
+    }
+
+  if (me() == 0) {
+      lock_.initialize();
+      lock_ = 1;
+      char * stringrep = lock_.stringrep();
+      int length = strlen(stringrep) + 1;
+      msg_->bcast(&length, 1);
+      msg_->bcast(stringrep, length);
+#ifdef DEBUG
+      ExEnv::outn() << scprintf("%d: sent initialize string of \"%s\" (%d)\n",
+                       me(), stringrep, length);
+      ExEnv::outn().flush();
+#endif // DEBUG
+      delete[] stringrep;
+      for (i=0; i<n(); i++) {
+          update_[i].initialize();
+          char * stringrep = update_[i].stringrep();
+          int length = strlen(stringrep) + 1;
+          msg_->bcast(&length, 1);
+          msg_->bcast(stringrep, length);
+#ifdef DEBUG
+          ExEnv::outn() << scprintf("%d: sent initialize string of \"%s\" (%d) for %d\n",
+                           me(), stringrep, length, i);
+          ExEnv::outn().flush();
+#endif // DEBUG
+          delete[] stringrep;
+        }
+    }
+  else {
+      int length;
+      msg_->bcast(&length, 1);
+      char * stringrep = new char[length];
+      msg_->bcast(stringrep, length);
+#ifdef DEBUG
+      ExEnv::outn() << scprintf("%d: got initialize string of \"%s\" (%d)\n",
+                       me(), stringrep, length);
+      ExEnv::outn().flush();
+#endif // DEBUG
+      lock_.initialize(stringrep);
+      delete[] stringrep;
+      for (i=0; i<n(); i++) {
+          msg_->bcast(&length, 1);
+          stringrep = new char[length];
+          msg_->bcast(stringrep, length);
+#ifdef DEBUG
+          ExEnv::outn() << scprintf("%d: got initialize string of \"%s\" (%d) for %d\n",
+                           me(), stringrep, length, i);
+          ExEnv::outn().flush();
+#endif // DEBUG
+          update_[i].initialize(stringrep);
+          delete[] stringrep;
+        }
+    }
+
+  // get an initial starting address on node 0
+  int ntry = 20;
+  int itry;
+  void *address;
+  if (me() == 0) {
+      address = 0;
+      itry = 0;
+      while (attach_memory(address,size) && itry < ntry) {
+          if (address == 0) address = memory_;
+          else address = (void*) &((char*)address)[0x1000000];
+          itry++;
+        }
+      if (itry == ntry) {
+          ExEnv::errn() << "ShmMemoryGrp: ntry exhausted on node 0" << endl;
+          abort();
+        }
+      // detach again, since we all try together below
+      detach_memory();
+    }
+
+  msg_->bcast(shmid_, nregion_);
+  msg_->raw_bcast((void*)&memory_, sizeof(void*));
+
+  address = memory_;
+  itry = 0;
+  int fail;
+  do {
+      fail = attach_memory(address,size);
+      msg_->max(fail);
+      if (fail) {
+          detach_memory();
+        }
+      address = (void*) &((char*)address)[0x1000000];
+      itry++;
+    } while(fail && itry < ntry);
+  if (itry == ntry) {
+      ExEnv::errn() << "ShmMemoryGrp: ntry exhausted on node " << me()
+           << " on joint attach phase" << endl;
+      abort();
+    }
+
+  if (me() == 0) {
+      // initialize the pool
+      pool_ = new(memory_) Pool(poolallocation);
+      rangelock_ = new(pool_->allocate(sizeof(RangeLock))) RangeLock(pool_);
+    }
+
+  msg_->raw_bcast((void*)&pool_, sizeof(void*));
+  msg_->raw_bcast((void*)&rangelock_, sizeof(void*));
+
+  if (debug_) {
+      ExEnv::outn() << scprintf("%d: memory_ = 0x%x shmid_[0] = %d\n",
+                       me(), memory_, shmid_[0]);
+    }
+
+  data_ = (void *) &((char*)memory_)[poolallocation];
+}
+
+void *
+ShmMemoryGrp::localdata()
+{
+  return &((char*)data_)[distsize_to_size(localoffset())];
+}
+
+void
+ShmMemoryGrp::cleanup()
+{
+
+  if (memory_) {
+      for (int i=0; i<nregion_; i++) {
+          shmdt((SHMTYPE)attach_address_[i]);
+        }
+
+      msg_->sync();
+
+      if (me() == 0) {
+          for (int i=0; i<nregion_; i++) {
+              shmctl(shmid_[i],IPC_RMID,0);
+            }
+        }
+      memory_ = 0;
+    }
+
+  delete[] update_;
+  update_ = 0;
+
+  nregion_ = 0;
+  delete[] shmid_;
+  delete[] attach_address_;
+  shmid_ = 0;
+  attach_address_ = 0;
+}
+
+ShmMemoryGrp::~ShmMemoryGrp()
+{
+  cleanup();
+
+#ifdef DEBUG
+  ExEnv::outn() << scprintf("msg_->nreference() = %d\n", msg_->nreference());
+  ExEnv::outn().flush();
+#endif // DEBUG
+  msg_ = 0;
+}
+
+void *
+ShmMemoryGrp::obtain_readwrite(distsize_t offset, int size)
+{
+  if (offset + size > totalsize()) {
+      ExEnv::errn() << scprintf("ShmMemoryGrp::obtain_readwrite: arg out of range\n");
+      abort();
+    }
+
+#if SIMPLE_LOCK
+  obtain_lock();
+#else // SIMPLE_LOCK
+#ifdef DEBUG
+  ExEnv::outn() << scprintf("%d: clear_release_count\n", me());
+  ExEnv::outn().flush();
+#endif // DEBUG
+  clear_release_count();
+#ifdef DEBUG
+  ExEnv::outn() << scprintf("%d: obtain_lock\n", me());
+  ExEnv::outn().flush();
+#endif // DEBUG
+  obtain_lock();
+#ifdef DEBUG
+  ExEnv::outn() << scprintf("%d: checkeq\n", me());
+  ExEnv::outn().flush();
+#endif
+  while (!rangelock_->checkeq(offset, offset + size, 0)) {
+#ifdef DEBUG
+      ExEnv::outn() << scprintf("%d: range not zero -- waiting for release\n", me());
+      ExEnv::outn().flush();
+#endif // DEBUG
+      //rangelock_->print();
+      release_lock();
+      wait_for_release();
+      obtain_lock();
+    }
+  rangelock_->decrement(offset, offset + size);
+#ifdef DEBUG
+  ExEnv::outn() << scprintf("%d: after obtain write\n", me());
+  ExEnv::outn().flush();
+  //rangelock_->print();
+#endif // DEBUG
+  release_lock();
+#endif // SIMPLE_LOCK
+
+  return &((char*)data_)[distsize_to_size(offset)];
+}
+
+void *
+ShmMemoryGrp::obtain_readonly(distsize_t offset, int size)
+{
+  if (offset + size > totalsize()) {
+      ExEnv::errn() << scprintf("ShmMemoryGrp::obtain_readonly: arg out of range\n");
+      abort();
+    }
+
+  return &((char*)data_)[distsize_to_size(offset)];
+}
+
+void *
+ShmMemoryGrp::obtain_writeonly(distsize_t offset, int size)
+{
+  if (offset + size > totalsize()) {
+      ExEnv::errn() << scprintf("ShmMemoryGrp::obtain_writeonly: arg out of range\n");
+      abort();
+    }
+
+  return &((char*)data_)[distsize_to_size(offset)];
+}
+
+void
+ShmMemoryGrp::release_readonly(void *data, distsize_t offset, int size)
+{
+}
+
+void
+ShmMemoryGrp::release_writeonly(void *data, distsize_t offset, int size)
+{
+}
+
+void
+ShmMemoryGrp::release_readwrite(void *data, distsize_t offset, int size)
+{
+#if SIMPLE_LOCK
+  release_lock();
+#else // SIMPLE_LOCK
+  obtain_lock();
+  rangelock_->increment(offset, offset + size);
+  note_release();
+#ifdef DEBUG
+  ExEnv::outn() << scprintf("%d: after release write\n", me());
+  //rangelock_->print();
+  ExEnv::outn().flush();
+#endif // DEBUG
+  release_lock();
+#endif // SIMPLE_LOCK
+}
+
+void
+ShmMemoryGrp::obtain_lock()
+{
+#ifdef DEBUG
+  ExEnv::outn() << scprintf("%d: lock val = %d\n", me(), lock_.val());
+  ExEnv::outn().flush();
+#endif // DEBUG
+  lock_--;
+#ifdef DEBUG
+  ExEnv::outn() << scprintf("%d: lock decremented\n", me());
+  ExEnv::outn().flush();
+#endif // DEBUG
+}
+
+void
+ShmMemoryGrp::release_lock()
+{
+  lock_++;
+#ifdef DEBUG
+  ExEnv::outn() << scprintf("%d: incremented lock\n", me());
+  ExEnv::outn().flush();
+#endif // DEBUG
+}
+
+void
+ShmMemoryGrp::note_release()
+{
+  for (int i=0; i<n(); i++) {
+      update_[i]++;
+    }
+#ifdef DEBUG
+  ExEnv::outn() << scprintf("%d: incremented release flags\n", me());
+  ExEnv::outn().flush();
+#endif // DEBUG
+}
+
+void
+ShmMemoryGrp::wait_for_release()
+{
+#ifdef DEBUG
+  ExEnv::outn() << scprintf("%d: decrementing release flag\n", me());
+  ExEnv::outn().flush();
+#endif // DEBUG
+  update_[me()]--;
+#ifdef DEBUG
+  ExEnv::outn() << scprintf("%d: decremented release flag\n", me());
+  ExEnv::outn().flush();
+#endif // DEBUG
+}
+
+void
+ShmMemoryGrp::clear_release_count()
+{
+  update_[me()] = 0;
+#ifdef DEBUG
+  ExEnv::outn() << scprintf("%d: clearing release count\n", me());
+  ExEnv::outn().flush();
+#endif // DEBUG
+}
+
+void
+ShmMemoryGrp::print(ostream &o) const
+{
+  MemoryGrp::print(o);
+}
+
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memshm.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memshm.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memshm.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,96 @@
+//
+// memshm.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_memshm_h
+#define _util_group_memshm_h
+
+#include <sys/types.h>
+#include <sys/ipc.h>
+#include <sys/shm.h>
+#include <iostream>
+
+#include <util/group/globcnt.h>
+#include <util/group/memmsg.h>
+
+#include <util/group/rnglock.h>
+
+namespace sc {
+
+/** The ShmMemoryGrp concrete class provides an implementation of
+MsgMemoryGrp.  It uses SYSV IPC to provided shared memory in a system that
+provide shared memory in hardware.  It is very fast and reliable. */
+class ShmMemoryGrp: public MsgMemoryGrp {
+  private:
+    int nregion_;
+    int *shmid_;
+    void **attach_address_;
+    GlobalCounter lock_;
+    GlobalCounter *update_;
+    void *data_;
+    void *memory_;
+    Pool *pool_;
+    RangeLock *rangelock_; // the locks_ member of the base class is ignored
+
+    void clear_release_count();
+    void wait_for_release();
+    void note_release();
+    void obtain_lock();
+    void release_lock();
+
+    void cleanup();
+    int attach_memory(void *ataddress, int size);
+    void detach_memory();
+  public:
+    ShmMemoryGrp(const Ref<MessageGrp>& msg);
+    ShmMemoryGrp(const Ref<KeyVal>&);
+    ~ShmMemoryGrp();
+
+    void set_localsize(size_t);
+    void *localdata();
+
+    void *obtain_readwrite(distsize_t offset, int size);
+    void *obtain_readonly(distsize_t offset, int size);
+    void *obtain_writeonly(distsize_t offset, int size);
+    void release_readonly(void *data, distsize_t offset, int size);
+    void release_writeonly(void *data, distsize_t offset, int size);
+    void release_readwrite(void *data, distsize_t offset, int size);
+
+    void print(std::ostream &o = ExEnv::out0()) const;
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memtest.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memtest.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memtest.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,437 @@
+//
+// memtest.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#include <math.h>
+#include <util/misc/formio.h>
+#include <util/misc/bug.h>
+#include <util/group/message.h>
+#include <util/group/mstate.h>
+#include <util/group/hcube.h>
+#include <util/group/memshm.h>
+#ifdef HAVE_NX
+#  include <util/group/memipgon.h>
+#endif
+
+using namespace std;
+using namespace sc;
+
+// Force linkages:
+//#ifndef __PIC__
+#ifdef HAVE_SYSV_IPC
+#   include <util/group/messshm.h>
+    static ForceLink<ShmMessageGrp> fl0;
+#endif
+#ifdef HAVE_MPI
+#   include <util/group/messmpi.h>
+#   include <util/group/memmtmpi.h>
+    static ForceLink<MPIMessageGrp> fl2;
+    static ForceLink<MTMPIMemoryGrp> fl3;
+#endif
+//#endif
+
+#include <util/class/scexception.h>
+static const char * (sc::SCException::*force_except_link)() const
+    = &sc::SCException::description;
+
+// this is needed for debugging
+#ifdef HAVE_NX
+extern "C" {
+#include <nx.h>
+}
+#endif // HAVE_NX
+
+#ifdef HAVE_HRECV
+#  define DISABLE do { masktrap(1); cout.flush(); } while(0)
+#  define ENABLE do { cout.flush(); masktrap(0); } while(0)
+   extern "C" {
+       long masktrap(long state);
+     }
+#else
+#  define DISABLE
+#  define ENABLE
+#endif
+
+#define PRINTF(args) do { DISABLE; \
+                          cout << scprintf args; \
+                          cout.flush(); \
+                          ENABLE; \
+                         } while(0)
+
+void do_simple_tests(const Ref<MessageGrp>&,const Ref<MemoryGrp>&);
+void do_int_tests(const Ref<MessageGrp>&,const Ref<MemoryGrp>&);
+void do_double_tests(const Ref<MessageGrp>&,const Ref<MemoryGrp>&);
+void do_double2_tests(const Ref<MessageGrp>&,const Ref<MemoryGrp>&);
+
+int
+main(int argc, char**argv)
+{
+  Ref<MessageGrp> msg = MessageGrp::initial_messagegrp(argc, argv);
+
+  const char* input = SRCDIR "/memtest.in";
+  Ref<KeyVal> keyval = new ParsedKeyVal(input);
+
+  if (msg.null()) {
+      const char* keyword = "message";
+
+      if (argc >= 2) input = argv[1];
+      if (argc >= 3) keyword = argv[2];
+
+      msg << keyval->describedclassvalue(keyword);
+
+      if (msg.null()) {
+          cerr << scprintf("Couldn't initialize MessageGrp\n");
+          abort();
+        }
+    }
+
+// This causes problems for automated testing:
+//   // now set up the debugger
+//   Ref<Debugger> debugger;
+//   debugger << keyval->describedclassvalue(":debug");
+//   if (debugger.nonnull()) {
+//     debugger->set_exec(argv[0]);
+//     debugger->set_prefix(msg->me());
+//   }
+
+  keyval = 0;
+
+  msg->sync();
+
+  MessageGrp::set_default_messagegrp(msg);
+
+  Ref<MemoryGrp> mem = MemoryGrp::initial_memorygrp(argc, argv);
+  if (mem.nonnull())
+    MemoryGrp::set_default_memorygrp(mem);
+  else
+    mem = MemoryGrp::get_default_memorygrp();
+
+  do_simple_tests(msg, mem);
+
+  do_double_tests(msg, mem);
+  do_double2_tests(msg, mem);
+
+  do_int_tests(msg, mem);
+
+  return 0;
+}
+
+void
+do_simple_tests(const Ref<MessageGrp>&msg,
+                const Ref<MemoryGrp>&mem)
+{
+  mem->set_localsize(8);
+
+  cout << scprintf("Using memory group \"%s\".\n", mem->class_name());
+
+  mem->sync();
+  mem->set_localsize(0);
+}
+
+void
+do_int_tests(const Ref<MessageGrp>&msg,
+             const Ref<MemoryGrp>&mem)
+{
+  const int intbufsize = 10;
+
+  mem->set_localsize(intbufsize*sizeof(int));
+
+  cout << scprintf("Using memory group \"%s\".\n", mem->class_name());
+
+  //sleep(1);
+  cout.flush();
+  cout << scprintf("111111111111111111111111111111111\n");
+  cout.flush();
+  //sleep(1);
+
+  mem->sync();
+
+  //sleep(1);
+  cout.flush();
+  cout << scprintf("222222222222222222222222222222222\n");
+  cout.flush();
+  //sleep(1);
+
+  //mem->deactivate();
+  //sleep(1);
+  cout.flush();
+  cout << scprintf("333333333333333333333333333333333\n");
+  cout.flush();
+  //sleep(1);
+  //mem = 0;
+  //return;
+
+  PRINTF(("creating MemoryGrpBuf\n"));
+
+  MemoryGrpBuf<int> buf(mem);
+
+  PRINTF(("%d: obtaining writelock\n", mem->me()));
+  int *data = buf.writeonly(0, intbufsize);
+  PRINTF(("%d: releasing writelock\n", mem->me()));
+  int i;
+  for (i=0; i<intbufsize; i++) data[i] = 0;
+  buf.release();
+
+  PRINTF(("%d: obtaining readlock\n", mem->me()));
+  const int *cdata = buf.readonly(0, intbufsize);
+  PRINTF(("%d: releasing readlock\n", mem->me()));
+  buf.release();
+
+//   if (mem->me() == 0) {
+//       cdata = buf.readonly(0, intbufsize);
+//       for (i=0; i<intbufsize; i++) {
+//           cout << scprintf("data[%3d] = %4d\n", i, cdata[i]);
+//         }
+//       buf.release();
+//     }
+
+  PRINTF(("%d: syncing\n", mem->me()));
+  mem->sync();
+  PRINTF(("---------------------------------------------------------\n"));
+  mem->sync();
+  PRINTF(("%d: done syncing\n", mem->me()));
+
+  if (mem->me() == 0 || mem->me() == 1) {
+      int start[2];
+      int length[2];
+      start[0] = 0;
+      length[0] = intbufsize/2;
+      start[1] = length[0];
+      length[1] = intbufsize - length[0];
+      data = buf.readwrite(start[mem->me()], length[mem->me()]);
+      PRINTF(("%d: adding %d to [%d, %d)\n",
+              mem->me(), 10 * (mem->me()+1),
+              start[mem->me()], start[mem->me()]+length[mem->me()]));
+      for (i=0; i<buf.length(); i++) {
+          data[i] += 10 * (mem->me() + 1);
+        }
+      buf.release();
+      mem->sync();
+      PRINTF(("------------------------------------------------------\n"));
+      mem->sync();
+      data = buf.readwrite(start[1-mem->me()], length[1-mem->me()]);
+      PRINTF(("%d: adding %d to [%d, %d)\n",
+              mem->me(), 100 * (mem->me()+1),
+              start[1-mem->me()], start[1-mem->me()]+length[1-mem->me()]));
+      for (i=0; i<buf.length(); i++) {
+          data[i] += 100 * (mem->me() + 1);
+        }
+      buf.release();
+      mem->sync();
+      PRINTF(("------------------------------------------------------\n"));
+      mem->sync();
+    }
+  else {
+      mem->sync();
+      PRINTF(("------------------------------------------------------\n"));
+      mem->sync();
+      mem->sync();
+      PRINTF(("------------------------------------------------------\n"));
+      mem->sync();
+   }
+
+  if (mem->me() == 0) {
+      cdata = buf.readonly(0, intbufsize);
+      for (i=0; i<intbufsize; i++) {
+          PRINTF(("data[%3d] = %4d\n", i, cdata[i]));
+        }
+      buf.release();
+    }
+
+  PRINTF(("%d: syncing\n", mem->me()));
+  mem->sync();
+  PRINTF(("---------------------------------------------------------\n"));
+  mem->sync();
+
+  PRINTF(("%d: exiting\n", mem->me()));
+  PRINTF(("%d: syncing\n", mem->me()));
+  mem->sync();
+  PRINTF(("==========================================================\n"));
+  mem->sync();
+
+  mem->set_localsize(0);
+}
+
+void
+do_double_tests(const Ref<MessageGrp>&msg,
+                const Ref<MemoryGrp>&mem)
+{
+  PRINTF(("double tests entered\n"));
+
+  int i,j;
+
+  const int doublebufsize = 4;
+
+  mem->set_localsize(doublebufsize*sizeof(double));
+
+  cout << scprintf("Using memory group \"%s\".\n", mem->class_name());
+
+  mem->sync();
+
+  MemoryGrpBuf<double> dbuf(mem);
+
+  PRINTF(("%d: double tests mem = 0x%x\n", mem->me(), mem.pointer()));
+
+  double factor = 1.0;
+  for (i=0; i<mem->me(); i++) factor *= 10.0;
+
+  PRINTF(("%d: setting 5th digit\n", mem->me()));
+
+  double *data = dbuf.writeonly_on_node(0, doublebufsize);
+  for (i=0; i<doublebufsize; i++) {
+      data[i] = 10000.0 * (mem->me()+1) + 100000.0 * i;
+    }
+  dbuf.release();
+
+  PRINTF(("%d: 5th digit set\n", mem->me()));
+
+  double contrib[doublebufsize];
+  for (i=0; i<doublebufsize; i++) {
+      contrib[i] = (mem->me()+1) * factor;
+    }
+
+  mem->sync();
+  PRINTF(("---------------------------------------------------------\n"));
+  mem->sync();
+
+  PRINTF(("%d: starting sum reduction\n", mem->me()));
+
+  for (i=0; i<mem->n(); i++) {
+      mem->sum_reduction_on_node(contrib, mem->me(),
+                                 doublebufsize-mem->me(),
+                                 i);
+    }
+
+  PRINTF(("%d: done with sum reduction\n", mem->me()));
+
+  mem->sync();
+  PRINTF(("---------------------------------------------------------\n"));
+  mem->sync();
+
+  for (i=0; i<mem->n(); i++) {
+      mem->sync();
+      if (i==mem->me()) {
+          const double *cdata = dbuf.readonly_on_node(0, doublebufsize);
+          for (j=0; j<doublebufsize; j++) {
+              PRINTF(("%2d: %12.1f\n", i, cdata[j]));
+            }
+          dbuf.release();
+        }
+    }
+
+  mem->sync();
+  PRINTF(("---------------------------------------------------------\n"));
+  mem->sync();
+
+  for (i=0; i<mem->n(); i++) {
+      mem->sync();
+      if (i==mem->me()) {
+          const double *cdata = dbuf.readonly(0, doublebufsize*mem->n());
+          for (j=0; j<doublebufsize*mem->n(); j++) {
+              PRINTF(("%2d: data[%2d] = %12.1f\n", i, j, cdata[j]));
+            }
+          dbuf.release();
+        }
+    }
+
+  mem->sync();
+  PRINTF(("==========================================================\n"));
+  mem->sync();
+
+  mem->set_localsize(0);
+}
+
+void
+do_double2_tests(const Ref<MessageGrp>&msg,
+                 const Ref<MemoryGrp>&mem)
+{
+  PRINTF(("double2 tests entered\n"));
+
+  int i,j;
+
+  const int doublebufsize = 4;
+  mem->set_localsize(doublebufsize*sizeof(double));
+  cout << scprintf("Using memory group \"%s\".\n", mem->class_name());
+
+  mem->sync();
+
+  MemoryGrpBuf<double> dbuf(mem);
+
+  double *data = dbuf.writeonly_on_node(0, doublebufsize);
+  for (i=0; i<doublebufsize; i++) {
+      data[i] = 0.0;
+    }
+  dbuf.release();
+
+  int target = (mem->me() + 1)%mem->n();
+
+  double contrib[doublebufsize];
+  for (i=0; i<doublebufsize; i++) {
+      contrib[i] = mem->me()/200.0;
+    }
+
+  mem->sync();
+  PRINTF(("---------------------------------------------------------\n"));
+  mem->sync();
+
+  PRINTF(("%d: starting sum reduction\n", mem->me()));
+
+  for (i=0; i<200; i++) {
+      mem->sum_reduction_on_node(contrib, 0,
+                                 doublebufsize,
+                                 target);
+    }
+
+  PRINTF(("%d: done with sum reduction\n", mem->me()));
+
+  mem->sync();
+  PRINTF(("---------------------------------------------------------\n"));
+  mem->sync();
+
+  for (i=0; i<mem->n(); i++) {
+      mem->sync();
+      if (i==mem->me()) {
+          const double *cdata = dbuf.readonly(0, doublebufsize*mem->n());
+          for (j=0; j<doublebufsize*mem->n(); j++) {
+              PRINTF(("%2d: data[%2d] = %12.1f\n", i, j, cdata[j]));
+            }
+          dbuf.release();
+        }
+    }
+
+  mem->sync();
+  PRINTF(("==========================================================\n"));
+  mem->sync();
+
+  mem->set_localsize(0);
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/memtest.in
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/memtest.in	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/memtest.in	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,15 @@
+% Emacs should use -*- KeyVal -*- mode.
+
+debug<Debugger>:(
+  )
+
+procmessage<ProcMessageGrp>: ()
+
+shmmessage<ShmMessageGrp>: (
+  n = 2
+  )
+
+mpimessage<MPIMessageGrp>: (
+  )
+
+message = $:procmessage
Index: ThirdParty/mpqc_open/src/lib/util/group/message.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/message.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/message.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,44 @@
+//
+// message.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_message_cc
+#define _util_group_message_cc
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#include <util/group/message.h>
+
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/message.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/message.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/message.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,420 @@
+//
+// message.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_message_h
+#define _util_group_message_h
+
+#include <map>
+
+#include <math.h>
+#include <util/class/class.h>
+#include <util/state/state.h>
+#include <util/keyval/keyval.h>
+#include <util/group/topology.h>
+
+namespace sc {
+
+template <class T>
+class GrpReduce {
+  public:
+    virtual ~GrpReduce() {};
+    virtual void reduce(T*target, T*data, int n) = 0;
+};
+
+template <class T>
+class GrpSumReduce: public GrpReduce<T> {
+  public:
+    ~GrpSumReduce() {};
+    void reduce(T*target, T*data, int nelement);
+};
+
+template <class T>
+class GrpMinReduce: public GrpReduce<T> {
+  public:
+    ~GrpMinReduce() {};
+    void reduce(T*target, T*data, int nelement);
+};
+
+template <class T>
+class GrpMaxReduce: public GrpReduce<T> {
+  public:
+    ~GrpMaxReduce() {};
+    void reduce(T*target, T*data, int nelement);
+};
+
+template <class T>
+class GrpArithmeticAndReduce: public GrpReduce<T> {
+  public:
+    void reduce(T*target, T*data, int nelement);
+};
+
+template <class T>
+class GrpArithmeticOrReduce: public GrpReduce<T> {
+  public:
+    void reduce(T*target, T*data, int nelement);
+};
+
+template <class T>
+class GrpArithmeticXOrReduce: public GrpReduce<T> {
+  public:
+    void reduce(T*target, T*data, int nelement);
+};
+
+template <class T>
+class GrpProductReduce: public GrpReduce<T> {
+  public:
+    void reduce(T*target, T*data, int nelement);
+};
+
+template <class T>
+class GrpFunctionReduce: public GrpReduce<T> {
+  private:
+    void (*func_)(T*target,T*data,int nelement);
+  public:
+    GrpFunctionReduce(void(*func)(T*,T*,int)):func_(func) {}
+    void reduce(T*target, T*data, int nelement);
+};
+
+/** The MessageGrp abstract class provides
+    a mechanism for moving data and objects between
+    nodes in a parallel machine. */
+class MessageGrp: public DescribedClass {
+  private:
+    // These are initialized by the initialize() member (see below).
+    int me_;
+    int n_;
+    int nclass_;
+    int gop_max_;
+    std::map<ClassDescP,int> classdesc_to_index_;
+    ClassDescP *index_to_classdesc_;
+  protected:
+    /** The classdesc_to_index_ and index_to_classdesc_ arrays
+        cannot be initialized by the MessageGrp CTOR, because
+        the MessageGrp specialization has not yet been initialized
+        and communication is not available.  CTOR's of specializations
+        of MessageGrp must call the initialize member in their body
+        to complete the initialization process. */
+    void initialize(int me, int n);
+
+    Ref<MachineTopology> topology_;
+
+    int debug_;
+  public:
+    MessageGrp();
+    MessageGrp(const Ref<KeyVal>&);
+    virtual ~MessageGrp();
+    
+    /// Returns the number of processors.
+    int n() { return n_; }
+    /// Returns my processor number.  In the range [0,n()).
+    int me() { return me_; }
+
+    /** Returns a copy of this MessageGrp specialization that provides
+        an independent communication context. */
+    virtual Ref<MessageGrp> clone(void)=0;
+    
+    /** The default message group contains the primary message group to
+        be used by an application. */
+    static void set_default_messagegrp(const Ref<MessageGrp>&);
+    /// Returns the default message group.
+    static MessageGrp* get_default_messagegrp();
+
+    /** Create a message group.  This routine looks for a -messagegrp
+        argument, then the environmental variable MESSAGEGRP to decide which
+        specialization of MessageGrp would be appropriate.  The
+        argument to -messagegrp should be either string for a
+        ParsedKeyVal constructor or a classname.  If this returns
+        null, it is up to the programmer to create a MessageGrp. */
+    static MessageGrp* initial_messagegrp(int &argc, char** &argv);
+
+    /** Send messages sequentially to the target processor.
+        Similar members exist for each of the basic types. */
+    virtual void send(int target, const double* data, int ndata);
+    virtual void send(int target, const unsigned int* data, int ndata);
+    virtual void send(int target, const int* data, int ndata);
+    virtual void send(int target, const char* data, int nbyte);
+    virtual void send(int target, const unsigned char* data, int nbyte);
+    virtual void send(int target, const signed char* data, int nbyte);
+    virtual void send(int target, const short* data, int ndata);
+    virtual void send(int target, const long* data, int ndata);
+    virtual void send(int target, const float* data, int ndata);
+    void send(int target, double data) { send(target,&data,1); }
+    void send(int target, int data) { send(target,&data,1); }
+    virtual void raw_send(int target, const void* data, int nbyte) = 0;
+
+    /** Send typed messages to the target processor.
+        Similar members exist for each of the basic types. */
+    virtual void sendt(int target, int type, const double* data, int ndata);
+    virtual void sendt(int target, int type, const unsigned int* data, int ndata);
+    virtual void sendt(int target, int type, const int* data, int ndata);
+    virtual void sendt(int target, int type, const char* data, int nbyte);
+    virtual void sendt(int target, int type, const unsigned char* data, int nbyte);
+    virtual void sendt(int target, int type, const signed char* data, int nbyte);
+    virtual void sendt(int target, int type, const short* data, int ndata);
+    virtual void sendt(int target, int type, const long* data, int ndata);
+    virtual void sendt(int target, int type, const float* data, int ndata);
+    void sendt(int target, int type, double data) {sendt(target,type,&data,1);}
+    void sendt(int target, int type, int data) {sendt(target,type,&data,1);}
+    virtual void raw_sendt(int target, int type, const void* data, int nbyte) = 0;
+
+    /** Receive messages sent sequentually from the sender.
+        Similar members exist for each of the basic types. */
+    virtual void recv(int sender, double* data, int ndata);
+    virtual void recv(int sender, unsigned int* data, int ndata);
+    virtual void recv(int sender, int* data, int ndata);
+    virtual void recv(int sender, char* data, int nbyte);
+    virtual void recv(int sender, unsigned char* data, int nbyte);
+    virtual void recv(int sender, signed char* data, int nbyte);
+    virtual void recv(int sender, short* data, int ndata);
+    virtual void recv(int sender, long* data, int ndata);
+    virtual void recv(int sender, float* data, int ndata);
+    void recv(int sender, double& data) { recv(sender,&data,1); }
+    void recv(int sender, int& data) { recv(sender,&data,1); }
+    virtual void raw_recv(int sender, void* data, int nbyte) = 0;
+
+    /** Receive messages sent by type.
+        Similar members exist for each of the basic types. */
+    virtual void recvt(int type, double* data, int ndata);
+    virtual void recvt(int type, unsigned int* data, int ndata);
+    virtual void recvt(int type, int* data, int ndata);
+    virtual void recvt(int type, char* data, int nbyte);
+    virtual void recvt(int type, unsigned char* data, int nbyte);
+    virtual void recvt(int type, signed char* data, int nbyte);
+    virtual void recvt(int type, short* data, int ndata);
+    virtual void recvt(int type, long* data, int ndata);
+    virtual void recvt(int type, float* data, int ndata);
+    void recvt(int type, double& data) { recvt(type,&data,1); }
+    void recvt(int type, int& data) { recvt(type,&data,1); }
+    virtual void raw_recvt(int type, void* data, int nbyte) = 0;
+
+    /// Ask if a given typed message has been received.
+    virtual int probet(int type) = 0;
+
+    /** Do broadcasts of various types of data.
+        Similar members exist for each of the basic types. */
+    virtual void bcast(double* data, int ndata, int from = 0);
+    virtual void bcast(unsigned int* data, int ndata, int from = 0);
+    virtual void bcast(int* data, int ndata, int from = 0);
+    virtual void bcast(char* data, int nbyte, int from = 0);
+    virtual void bcast(unsigned char* data, int nbyte, int from = 0);
+    virtual void bcast(signed char* data, int nbyte, int from = 0);
+    virtual void bcast(short* data, int ndata, int from = 0);
+    virtual void bcast(long* data, int ndata, int from = 0);
+    virtual void bcast(float* data, int ndata, int from = 0);
+    virtual void raw_bcast(void* data, int nbyte, int from = 0);
+    void bcast(double& data, int from = 0) { bcast(&data, 1, from); }
+    void bcast(int& data, int from = 0) { bcast(&data, 1, from); }
+
+    /** Collect data distributed on the nodes to a big array replicated
+        on each node. */
+    virtual void raw_collect(const void *part, const int *lengths,
+                             void *whole, int bytes_per_datum=1);
+    void collect(const double *part, const int *lengths, double *whole);
+
+    /** Global sum reduction.
+        Similar members exist for each of the basic types. */
+    virtual void sum(double* data, int n, double* = 0, int target = -1);
+    virtual void sum(unsigned int* data, int n, unsigned int* = 0, int target = -1);
+    virtual void sum(int* data, int n, int* = 0, int target = -1);
+    virtual void sum(char* data, int n, char* = 0, int target = -1);
+    virtual void sum(unsigned char* data, int n,
+                     unsigned char* = 0, int target = -1);
+    virtual void sum(signed char* data, int n,
+                     signed char* = 0, int target = -1);
+    void sum(double& data) { sum(&data, 1); }
+    void sum(int& data) { sum(&data, 1); }
+    /** Global maximization.
+        Similar members exist for each of the basic types. */
+    virtual void max(double* data, int n, double* = 0, int target = -1);
+    virtual void max(int* data, int n, int* = 0, int target = -1);
+    virtual void max(unsigned int* data, int n, unsigned int* = 0, int target = -1);
+    virtual void max(char* data, int n, char* = 0, int target = -1);
+    virtual void max(unsigned char* data, int n,
+                     unsigned char* = 0, int target = -1);
+    virtual void max(signed char* data, int n,
+                     signed char* = 0, int target = -1);
+    void max(double& data) { max(&data, 1); }
+    void max(int& data) { max(&data, 1); }
+    /** Global minimization.
+        Similar members exist for each of the basic types. */
+    virtual void min(double* data, int n, double* = 0, int target = -1);
+    virtual void min(int* data, int n, int* = 0, int target = -1);
+    virtual void min(unsigned int* data, int n, unsigned int* = 0, int target = -1);
+    virtual void min(char* data, int n, char* = 0, int target = -1);
+    virtual void min(unsigned char* data, int n,
+                     unsigned char* = 0, int target = -1);
+    virtual void min(signed char* data, int n,
+                     signed char* = 0, int target = -1);
+    void min(double& data) { min(&data, 1); }
+    void min(int& data) { min(&data, 1); }
+    /** Global generic reduction.
+        Similar members exist for each of the basic types. */
+    virtual void reduce(double*, int n, GrpReduce<double>&,
+                        double*scratch = 0, int target = -1);
+    virtual void reduce(int*, int n, GrpReduce<int>&,
+                        int*scratch = 0, int target = -1);
+    virtual void reduce(unsigned int*, int n, GrpReduce<unsigned int>&,
+                        unsigned int*scratch = 0, int target = -1);
+    virtual void reduce(char*, int n, GrpReduce<char>&,
+                        char*scratch = 0, int target = -1);
+    virtual void reduce(unsigned char*, int n, GrpReduce<unsigned char>&,
+                        unsigned char*scratch = 0, int target = -1);
+    virtual void reduce(signed char*, int n, GrpReduce<signed char>&,
+                        signed char*scratch = 0, int target = -1);
+    virtual void reduce(short*, int n, GrpReduce<short>&,
+                        short*scratch = 0, int target = -1);
+    virtual void reduce(float*, int n, GrpReduce<float>&,
+                        float*scratch = 0, int target = -1);
+    virtual void reduce(long*, int n, GrpReduce<long>&,
+                        long*scratch = 0, int target = -1);
+    void reduce(double& data, GrpReduce<double>& r) { reduce(&data, 1, r); }
+    void reduce(int& data, GrpReduce<int>& r) { reduce(&data, 1, r); }
+
+    /// Synchronize all of the processors.
+    virtual void sync();
+
+    /// Return the MachineTopology object.
+    Ref<MachineTopology> topology() { return topology_; }
+
+    /** Each message group maintains an association of ClassDesc with
+        a global index so SavableState information can be sent between
+        nodes without needing to send the classname and look up the
+        ClassDesc with each transfer.  These routines return information
+        about that mapping. */
+    int classdesc_to_index(const ClassDesc*);
+    const ClassDesc* index_to_classdesc(int);
+    int nclass() const { return nclass_; }
+};
+
+struct message_struct {
+  void *buf;
+  int size;
+  int type;
+  struct message_struct *p;
+  };
+typedef struct message_struct message_t;
+
+
+/** ProcMessageGrp provides a concrete specialization
+    of MessageGrp that supports only one node. */
+class ProcMessageGrp: public MessageGrp {
+  private:
+    // Messages are stored in these linked lists
+    message_t *sync_messages;
+    message_t *type_messages;
+
+    void sendit(message_t *& messages, int dest, int msgtype, const void* buf, int bytes);
+    void recvit(message_t *& messages, int source, int type, void* buf, int bytes,
+                int& last_size, int& last_type);
+        
+  public:
+    ProcMessageGrp();
+    ProcMessageGrp(const Ref<KeyVal>&);
+    ~ProcMessageGrp();
+
+    Ref<MessageGrp> clone(void);
+    
+    void raw_send(int target, const void* data, int nbyte);
+    void raw_sendt(int target, int type, const void* data, int nbyte);
+    void raw_recv(int sender, void* data, int nbyte);
+    void raw_recvt(int type, void* data, int nbyte);
+    void raw_bcast(void* data, int nbyte, int from);
+    int probet(int type);
+    void sync();
+};
+
+/** Uses integer message types to send and receive messages.
+    Message group specializations that use the MPI library
+    and the Paragon NX can be conveniently implemented in terms
+    of this. */
+class intMessageGrp: public MessageGrp {
+  protected:
+    int msgtype_nbit; // the total number of bits available
+    int ctl_nbit; // control information bits
+    int seq_nbit; // sequence information bits
+    int typ_nbit; // type information bits
+    int src_nbit; // source information bits
+
+    // Masks for the fields in the type.
+    int ctl_mask;
+    int seq_mask;
+    int typ_mask;
+    int src_mask;
+
+    // Shifts to construct a message type.
+    int ctl_shift;
+    int seq_shift;
+    int typ_shift;
+    int src_shift;
+
+    int msgtype_typ(int msgtype);
+    int typ_msgtype(int usrtype);
+    int seq_msgtype(int source, int seq);
+
+    // The next sequence number for each node is stored in these.
+    int *source_seq;
+    int *target_seq;
+    
+    /// Must be implemented by specializations.
+    virtual void basic_send(int target, int type, const void* data, int nbyte) = 0;
+    /// Must be implemented by specializations.
+    virtual void basic_recv(int type, void* data, int nbyte) = 0;
+    /// Must be implemented by specializations.
+    virtual int basic_probe(int type) = 0;
+
+    intMessageGrp();
+    intMessageGrp(const Ref<KeyVal>&);
+
+    void initialize(int me, int n, int nbits);
+  public:
+    ~intMessageGrp();
+
+    void raw_send(int target, const void* data, int nbyte);
+    void raw_recv(int sender, void* data, int nbyte);
+    void raw_sendt(int target, int type, const void* data, int nbyte);
+    void raw_recvt(int type, void* data, int nbyte);
+
+    int probet(int);
+
+    int leftover_ctl_bits();
+};
+
+}
+
+#include <util/group/messaget.h>
+
+#endif
+
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/messaget.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/messaget.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/messaget.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,118 @@
+//
+// messaget.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_messaget_h
+#define _util_group_messaget_h
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#include <util/group/message.h>
+
+namespace sc {
+
+template <class T>
+void
+GrpSumReduce<T>::reduce(T*target, T*data, int nelement)
+{
+  for (int i=0; i<nelement; i++) {
+      target[i] += data[i];
+    }
+}
+
+template <class T>
+void
+GrpMinReduce<T>::reduce(T*target, T*data, int nelement)
+{
+  for (int i=0; i<nelement; i++) {
+      if (target[i] > data[i]) target[i] = data[i];
+    }
+}
+
+template <class T>
+void
+GrpMaxReduce<T>::reduce(T*target, T*data, int nelement)
+{
+  for (int i=0; i<nelement; i++) {
+      if (target[i] < data[i]) target[i] = data[i];
+    }
+}
+
+template <class T>
+void
+GrpArithmeticAndReduce<T>::reduce(T*target, T*data, int nelement)
+{
+  for (int i=0; i<nelement; i++) {
+      target[i] = target[i] & data[i];
+    }
+}
+
+template <class T>
+void
+GrpArithmeticOrReduce<T>::reduce(T*target, T*data, int nelement)
+{
+  for (int i=0; i<nelement; i++) {
+      target[i] = target[i] | data[i];
+    }
+}
+
+template <class T>
+void
+GrpArithmeticXOrReduce<T>::reduce(T*target, T*data, int nelement)
+{
+  for (int i=0; i<nelement; i++) {
+      target[i] = target[i] ^ data[i];
+    }
+}
+
+template <class T>
+void
+GrpProductReduce<T>::reduce(T*target, T*data, int nelement)
+{
+  for (int i=0; i<nelement; i++) {
+      target[i] *= data[i];
+    }
+}
+
+template <class T>
+void
+GrpFunctionReduce<T>::reduce(T*target, T*data, int nelement)
+{
+  (*func_)(target,data,nelement);
+}
+
+}
+
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/messimpl.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/messimpl.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/messimpl.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,649 @@
+//
+// messimpl.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#include <string.h>
+
+#include <util/misc/formio.h>
+#include <util/misc/exenv.h>
+
+#include <util/group/message.h>
+
+#include <util/group/topology.h>
+#include <util/group/hcube.h>
+#ifdef HAVE_NX
+#  include <util/group/messpgon.h>
+#endif
+#ifdef HAVE_MPI
+#  define MPICH_SKIP_MPICXX
+#  include <mpi.h>
+#  include <util/group/messmpi.h>
+#endif
+
+#define DEFAULT_GOP_MAX 320000
+
+using namespace std;
+using namespace sc;
+
+static ClassDesc MessageGrp_cd(
+  typeid(MessageGrp),"MessageGrp",1,"public DescribedClass",
+  0, 0, 0);
+
+MessageGrp::MessageGrp(const Ref<KeyVal>& keyval):
+  me_(-1),
+  n_(-1),
+  index_to_classdesc_(0)
+{
+  gop_max_ = keyval->intvalue("gop_max");
+  if (keyval->error() != KeyVal::OK) gop_max_ = DEFAULT_GOP_MAX;
+  debug_ = keyval->booleanvalue("debug");
+}
+
+MessageGrp::MessageGrp():
+  me_(-1),
+  n_(-1),
+  index_to_classdesc_(0)
+{
+  gop_max_ = DEFAULT_GOP_MAX;
+  debug_ = 0;
+}
+
+MessageGrp::~MessageGrp()
+{
+  if (index_to_classdesc_) delete[] index_to_classdesc_;
+}
+
+static Ref<MessageGrp> default_messagegrp;
+
+void
+MessageGrp::set_default_messagegrp(const Ref<MessageGrp>& grp)
+{
+  default_messagegrp = grp;
+}
+
+MessageGrp*
+MessageGrp::get_default_messagegrp()
+{
+  if (default_messagegrp.null()) {
+#if defined(HAVE_MPI) && defined(DEFAULT_MPI)
+      default_messagegrp = new MPIMessageGrp;
+#else
+      default_messagegrp = new ProcMessageGrp;
+#endif
+    }
+  return default_messagegrp.pointer();
+}
+
+MessageGrp *
+MessageGrp::initial_messagegrp(int &argc, char** &argv)
+{
+  MessageGrp *grp = 0;
+
+  char *keyval_string = 0;
+
+  // see if a message group is given on the command line
+  if (argc && argv) {
+      for (int i=0; i<argc; i++) {
+	  if (argv[i] && !strcmp(argv[i], "-messagegrp")) {
+              char *messagegrp_string = argv[i];
+              i++;
+              if (i >= argc) {
+                  ExEnv::errn() << "-messagegrp must be following by an argument"
+                       << endl;
+                  abort();
+                }
+              keyval_string = argv[i];
+              // move the messagegrp arguments to the end of argv
+              int j;
+              for (j=i+1; j<argc; j++) {
+                  argv[j-2] = argv[j];
+                }
+              argv[j++] = messagegrp_string;
+              argv[j++] = keyval_string;
+              // decrement argc to hide the last two arguments
+              argc -= 2;
+              break;
+            }
+        }
+    }
+
+  if (!keyval_string) {
+      // find out if the environment gives the containing message group
+      keyval_string = getenv("MESSAGEGRP");
+      if (keyval_string) {
+          if (!strncmp("MESSAGEGRP=", keyval_string, 11)) {
+              keyval_string = strchr(keyval_string, '=');
+            }
+          if (*keyval_string == '=') keyval_string++;
+        }
+    }
+
+  // if keyval input for a message group was found, then
+  // create it.
+  if (keyval_string) {
+      if (keyval_string[0] == '\0') return 0;
+      //ExEnv::outn() << "Creating MessageGrp from \"" << keyval_string << "\"" << endl;
+      Ref<ParsedKeyVal> strkv = new ParsedKeyVal();
+      strkv->parse_string(keyval_string);
+      Ref<DescribedClass> dc = strkv->describedclassvalue();
+      grp = dynamic_cast<MessageGrp*>(dc.pointer());
+      if (dc.null()) {
+          ExEnv::errn() << "initial_messagegrp: couldn't find a MessageGrp in "
+               << keyval_string << endl;
+          abort();
+        }
+      else if (!grp) {
+          ExEnv::errn() << "initial_messagegrp: wanted MessageGrp but got "
+               << dc->class_name() << endl;
+          abort();
+        }
+      // prevent an accidental delete
+      grp->reference();
+      strkv = 0;
+      dc = 0;
+      // accidental delete not a problem anymore since all smart pointers
+      // to grp are dead
+      grp->dereference();
+      return grp;
+    }
+
+#if defined(HAVE_MPI)
+  int mpiinited;
+#ifdef ALWAYS_USE_MPI
+  bool always_use_mpi = true;
+#else
+  bool always_use_mpi = false;
+#endif
+  MPI_Initialized(&mpiinited);
+  if (mpiinited || always_use_mpi) {
+      grp = new MPIMessageGrp(&argc,&argv);
+      return grp;
+  }
+#endif
+
+  return 0;
+}
+
+void
+MessageGrp::initialize(int me, int n)
+{
+  // This member is called by a CTOR and, ultimately, causes
+  // 'this' to be converted into a temporary Ref which causes
+  // this to be deleted (very bad), so reference 'this'
+  // (and dereference this down below).
+  this->reference();
+
+  if (topology_.null()) {
+      topology_ = new HypercubeTopology();
+    }
+
+  int i;
+  std::map<std::string,ClassDescP>::iterator J;
+  
+  me_ = me;
+  n_ = n;
+
+  // get all of the classes known on this node
+  std::map<std::string,ClassDescP>& classes = ClassDesc::all();
+
+  // Keeps count of how many classes are known.
+  int iclass = 0;
+
+  for (i=0; i<n; i++) {
+      if (i==me) {
+          // Find out how many of my classes are not yet in the
+          // classdesc to index map.
+          int n_new_class = 0;
+          int buffer_size = 0;
+          for (J=classes.begin(); J!=classes.end(); J++) {
+              if (classdesc_to_index_.find(J->second) == classdesc_to_index_.end()) {
+                  n_new_class++;
+                  buffer_size += strlen(J->second->name()) + 1;
+                }
+            }
+          char* buffer = new char[buffer_size];
+          char* currentbuffer = buffer;
+          for (J=classes.begin(); J!=classes.end(); J++) {
+              if (classdesc_to_index_.find(J->second) == classdesc_to_index_.end()) {
+                  classdesc_to_index_[J->second] = iclass;
+                  iclass++;
+#ifdef DEBUG
+                  ExEnv::outn() << scprintf("node %d adding class %d = \"%s\"\n",
+                                   me, iclass, J->second->name());
+#endif
+                  strcpy(currentbuffer,J->second->name());
+                  currentbuffer += strlen(J->second->name()) + 1;
+                }
+            }
+#ifdef DEBUG
+          ExEnv::outn() << scprintf("node %d bcast n_new_class = %d\n",me,n_new_class);
+#endif
+          bcast(&n_new_class,1,i);
+#ifdef DEBUG
+          ExEnv::outn() << scprintf("node %d finished bcast\n",me);
+#endif
+          if (n_new_class) {
+              bcast(&buffer_size,1,i);
+              bcast(buffer,buffer_size,i);
+            }
+          delete[] buffer;
+        }
+      else {
+          int j;
+          // Get new classnames and indices from node i.
+          int n_new_class;
+#ifdef DEBUG
+          ExEnv::outn() << scprintf("node %d begin recv bcast\n",me);
+#endif
+          bcast(&n_new_class,1,i);
+#ifdef DEBUG
+          ExEnv::outn() << scprintf("node %d recv bcast n_new_class = %d\n",
+                           me,n_new_class);
+#endif
+          if (n_new_class) {
+              int buffer_size;
+              bcast(&buffer_size,1,i);
+              char* buffer = new char[buffer_size];
+              char* currentbuffer = buffer;
+              bcast(buffer,buffer_size,i);
+              for (j=0; j<n_new_class; j++) {
+                  ClassDescP cd = ClassDesc::name_to_class_desc(currentbuffer);
+                  if (cd) {
+#ifdef DEBUG
+                      ExEnv::outn() << scprintf("node %d adding \"%s\"\n",
+                                       me, currentbuffer);
+#endif
+                      classdesc_to_index_[cd] = iclass;
+                    }
+#ifdef DEBUG
+                  else {
+                      ExEnv::outn() << scprintf("node %d failed to add \"%s\"\n",
+                                       me, currentbuffer);
+                    }
+#endif
+                  iclass++;
+                  // advance the currentbuffer to the next classname
+                  while(*currentbuffer != '\0') currentbuffer++;
+                  currentbuffer++;
+                }
+              delete[] buffer;
+            }
+        }
+    }
+  nclass_ = iclass;
+
+  // Construct the mapping of index to classdesc.
+  index_to_classdesc_ = new ClassDescP[iclass];
+  for (i=0; i<nclass_; i++) {
+      index_to_classdesc_[i] = 0;
+    }
+  for (J=classes.begin(); J!=classes.end(); J++) {
+      if (classdesc_to_index_.find(J->second) != classdesc_to_index_.end()) {
+          index_to_classdesc_[classdesc_to_index_[J->second]] = J->second;
+        }
+    }
+
+  this->dereference();
+}
+
+// Sequential send routines
+
+void
+MessageGrp::send(int target, const double* data, int ndata)
+{
+  raw_send(target, data, ndata*sizeof(double));
+}
+void
+MessageGrp::send(int target, const short* data, int ndata)
+{
+  raw_send(target, data, ndata*sizeof(short));
+}
+void
+MessageGrp::send(int target, const long* data, int ndata)
+{
+  raw_send(target, data, ndata*sizeof(long));
+}
+void
+MessageGrp::send(int target, const float* data, int ndata)
+{
+  raw_send(target, data, ndata*sizeof(float));
+}
+void
+MessageGrp::send(int target, const unsigned int* data, int ndata)
+{
+  raw_send(target, data, ndata*sizeof(int));
+}
+void
+MessageGrp::send(int target, const int* data, int ndata)
+{
+  raw_send(target, data, ndata*sizeof(int));
+}
+void
+MessageGrp::send(int target, const char* data, int ndata)
+{
+  raw_send(target, data, ndata);
+}
+void
+MessageGrp::send(int target, const unsigned char* data, int ndata)
+{
+  raw_send(target, data, ndata);
+}
+void
+MessageGrp::send(int target, const signed char* data, int ndata)
+{
+  raw_send(target, data, ndata);
+}
+
+// Sequential receive routines
+
+void
+MessageGrp::recv(int sender, double* data, int ndata)
+{
+  raw_recv(sender, data, ndata*sizeof(double));
+}
+void
+MessageGrp::recv(int sender, short* data, int ndata)
+{
+  raw_recv(sender, data, ndata*sizeof(short));
+}
+void
+MessageGrp::recv(int sender, long* data, int ndata)
+{
+  raw_recv(sender, data, ndata*sizeof(long));
+}
+void
+MessageGrp::recv(int sender, float* data, int ndata)
+{
+  raw_recv(sender, data, ndata*sizeof(float));
+}
+void
+MessageGrp::recv(int sender, unsigned int* data, int ndata)
+{
+  raw_recv(sender, data, ndata*sizeof(int));
+}
+void
+MessageGrp::recv(int sender, int* data, int ndata)
+{
+  raw_recv(sender, data, ndata*sizeof(int));
+}
+void
+MessageGrp::recv(int sender, char* data, int ndata)
+{
+  raw_recv(sender, data, ndata);
+}
+void
+MessageGrp::recv(int sender, unsigned char* data, int ndata)
+{
+  raw_recv(sender, data, ndata);
+}
+void
+MessageGrp::recv(int sender, signed char* data, int ndata)
+{
+  raw_recv(sender, data, ndata);
+}
+
+// Typed send routines
+
+void
+MessageGrp::sendt(int target, int type, const double* data, int ndata)
+{
+  raw_sendt(target, type, data, ndata*sizeof(double));
+}
+void
+MessageGrp::sendt(int target, int type, const short* data, int ndata)
+{
+  raw_sendt(target, type, data, ndata*sizeof(short));
+}
+void
+MessageGrp::sendt(int target, int type, const long* data, int ndata)
+{
+  raw_sendt(target, type, data, ndata*sizeof(long));
+}
+void
+MessageGrp::sendt(int target, int type, const float* data, int ndata)
+{
+  raw_sendt(target, type, data, ndata*sizeof(float));
+}
+void
+MessageGrp::sendt(int target, int type, const unsigned int* data, int ndata)
+{
+  raw_sendt(target, type, data, ndata*sizeof(int));
+}
+void
+MessageGrp::sendt(int target, int type, const int* data, int ndata)
+{
+  raw_sendt(target, type, data, ndata*sizeof(int));
+}
+void
+MessageGrp::sendt(int target, int type, const char* data, int ndata)
+{
+  raw_sendt(target, type, data, ndata);
+}
+void
+MessageGrp::sendt(int target, int type, const unsigned char* data, int ndata)
+{
+  raw_sendt(target, type, data, ndata);
+}
+void
+MessageGrp::sendt(int target, int type, const signed char* data, int ndata)
+{
+  raw_sendt(target, type, data, ndata);
+}
+
+// Typed receive routines
+
+void
+MessageGrp::recvt(int type, double* data, int ndata)
+{
+  raw_recvt(type, data, ndata*sizeof(double));
+}
+void
+MessageGrp::recvt(int type, short* data, int ndata)
+{
+  raw_recvt(type, data, ndata*sizeof(short));
+}
+void
+MessageGrp::recvt(int type, long* data, int ndata)
+{
+  raw_recvt(type, data, ndata*sizeof(long));
+}
+void
+MessageGrp::recvt(int type, float* data, int ndata)
+{
+  raw_recvt(type, data, ndata*sizeof(float));
+}
+void
+MessageGrp::recvt(int type, unsigned int* data, int ndata)
+{
+  raw_recvt(type, data, ndata*sizeof(int));
+}
+void
+MessageGrp::recvt(int type, int* data, int ndata)
+{
+  raw_recvt(type, data, ndata*sizeof(int));
+}
+void
+MessageGrp::recvt(int type, char* data, int ndata)
+{
+  raw_recvt(type, data, ndata);
+}
+void
+MessageGrp::recvt(int type, unsigned char* data, int ndata)
+{
+  raw_recvt(type, data, ndata);
+}
+void
+MessageGrp::recvt(int type, signed char* data, int ndata)
+{
+  raw_recvt(type, data, ndata);
+}
+
+// Broadcast operations
+
+void
+MessageGrp::bcast(double*data, int ndata, int from)
+{
+  raw_bcast(data, ndata*sizeof(double), from);
+}
+void
+MessageGrp::bcast(short*data, int ndata, int from)
+{
+  raw_bcast(data, ndata*sizeof(short), from);
+}
+void
+MessageGrp::bcast(long*data, int ndata, int from)
+{
+  raw_bcast(data, ndata*sizeof(long), from);
+}
+void
+MessageGrp::bcast(float*data, int ndata, int from)
+{
+  raw_bcast(data, ndata*sizeof(float), from);
+}
+void
+MessageGrp::bcast(unsigned int*data, int ndata, int from)
+{
+  raw_bcast(data, ndata*sizeof(int), from);
+}
+void
+MessageGrp::bcast(int*data, int ndata, int from)
+{
+  raw_bcast(data, ndata*sizeof(int), from);
+}
+void
+MessageGrp::bcast(char*data, int ndata, int from)
+{
+  raw_bcast(data, ndata, from);
+}
+void
+MessageGrp::bcast(unsigned char*data, int ndata, int from)
+{
+  raw_bcast(data, ndata, from);
+}
+void
+MessageGrp::bcast(signed char*data, int ndata, int from)
+{
+  raw_bcast(data, ndata, from);
+}
+
+// Global classdesc indices
+
+int
+MessageGrp::classdesc_to_index(const ClassDesc* cdptr)
+{
+  if (classdesc_to_index_.find((ClassDesc*)cdptr) != classdesc_to_index_.end()) {
+      return classdesc_to_index_[(ClassDesc*)cdptr];
+    }
+  else {
+      return -1;
+    }
+}
+
+const ClassDesc*
+MessageGrp::index_to_classdesc(int index)
+{
+  if (index < 0 || index >= nclass_) {
+      return 0;
+    }
+  else {
+      return index_to_classdesc_[index];
+    }
+}
+
+void
+MessageGrp::raw_bcast(void* data, int nbyte, int from)
+{
+  int nbyte_actual = nbyte;
+  int tgop_max = nbyte;
+  if (gop_max_ != 0) {
+      tgop_max = gop_max_;
+      gop_max_ = 0;
+      bcast(nbyte_actual,from);
+      gop_max_ = tgop_max;
+    }
+  for (int idat=0; idat<nbyte_actual; idat+=tgop_max) {
+      int ndat = (idat+tgop_max>nbyte_actual)?(nbyte_actual-idat):tgop_max;
+      Ref<GlobalMsgIter> i(topology_->global_msg_iter(this, from));
+      for (i->forwards(); !i->done(); i->next()) {
+          if (i->send()) {
+              raw_send(i->sendto(), &((char*)data)[idat], ndat);
+            }
+          if (i->recv()) {
+              raw_recv(i->recvfrom(), &((char*)data)[idat], ndat);
+            }
+        }
+    }
+}
+
+void
+MessageGrp::sync()
+{
+  Ref<GlobalMsgIter> i(topology_->global_msg_iter(this, 0));
+
+  for (i->backwards(); !i->done(); i->next()) {
+      if (i->send()) {
+          raw_send(i->sendto(), 0, 0);
+        }
+      if (i->recv()) {
+          raw_recv(i->recvfrom(), 0, 0);
+        }
+    }
+  
+  for (i->forwards(); !i->done(); i->next()) {
+      if (i->send()) {
+          raw_send(i->sendto(), 0, 0);
+        }
+      if (i->recv()) {
+          raw_recv(i->recvfrom(), 0, 0);
+        }
+    }
+}
+
+void
+MessageGrp::collect(const double *part, const int *lengths, double *whole)
+{
+  raw_collect(part,lengths,whole,sizeof(double));
+}
+
+void
+MessageGrp::raw_collect(const void *part, const int *lengths, void *whole,
+                        int bytes_per_datum)
+{
+  int offset = 0;
+  for (int i=0; i<n_; i++) {
+      int nbytes = lengths[i];
+      if (i==me_) memcpy(&((char*)whole)[offset], part, nbytes);
+      raw_bcast(&((char*)whole)[offset], nbytes, i);
+      offset += nbytes;
+    }
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/messint.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/messint.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/messint.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,181 @@
+//
+// messint.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#include <util/misc/formio.h>
+#include <util/group/message.h>
+
+using namespace std;
+using namespace sc;
+
+static ClassDesc intMessageGrp_cd(
+  typeid(intMessageGrp),"intMessageGrp",1,"public MessageGrp",
+  0, 0, 0);
+
+intMessageGrp::intMessageGrp():
+  ctl_nbit(2),
+  ctl_mask(0x3)
+{
+}
+
+intMessageGrp::intMessageGrp(const Ref<KeyVal>& keyval):
+  MessageGrp(keyval),
+  ctl_nbit(2),
+  ctl_mask(0x3)
+{
+}
+
+intMessageGrp::~intMessageGrp()
+{
+  delete[] source_seq;
+  delete[] target_seq;
+}
+
+void
+intMessageGrp::initialize(int me, int n, int nbits)
+{
+  int i;
+  
+  // Initialize the arrays storing the next sequence number.
+  source_seq = new int[n];
+  target_seq = new int[n];
+  for (i=0; i<n; i++) {
+      source_seq[i] = 0;
+      target_seq[i] = 0;
+    }
+
+  // Find how many bits are needed to store the source information.
+  int tmp = 1;
+  for (i=1; tmp<n; i++) {
+      tmp = tmp<<1;
+    }
+  src_nbit = i;
+
+  // The remaining bits are for sequence information.
+  seq_nbit = nbits - ctl_nbit - src_nbit;
+  if (seq_nbit < 8) {
+      ExEnv::errn() << "intMessageGrp: not enough bits in underlying msgtype" << endl;
+      abort();
+    }
+
+  // The bits available to user defined messages overlap
+  // with the seq and the src bits.
+  typ_nbit = seq_nbit + src_nbit;
+
+  // Compute the shifts needed to construct a message type.
+  seq_shift = 0;
+  typ_shift = 0;
+  src_shift = seq_shift + seq_nbit;
+  ctl_shift = src_shift + src_nbit;
+
+  // Compute the masks for each field.
+  seq_mask = (1<<seq_nbit) - 1;
+  typ_mask = (1<<typ_nbit) - 1;
+  src_mask = (1<<src_nbit) - 1;
+
+  // Complete initialization of the base class.
+  MessageGrp::initialize(me, n);
+}
+
+int
+intMessageGrp::msgtype_typ(int msgtype)
+{
+  return msgtype>>typ_shift & typ_mask;
+}
+
+int
+intMessageGrp::typ_msgtype(int usrtype)
+{
+  return usrtype<<typ_shift | 1<<ctl_shift;
+}
+
+int
+intMessageGrp::seq_msgtype(int source, int seq)
+{
+  return source<<src_shift | seq<<seq_shift | 2<<ctl_shift;
+}
+
+void
+intMessageGrp::raw_send(int target, const void* data, int nbyte)
+{
+  int& seq = target_seq[target];
+  int msgtype = seq_msgtype(me(),seq);
+#ifdef DEBUG
+  ExEnv::outn() << scprintf("node %d sending to %d(%d) msgtype = %d\n",
+                   me(),target,seq,msgtype);
+#endif
+  basic_send(target, msgtype, data, nbyte);
+  if (seq >= seq_mask) seq = 0;
+  else seq++;
+}
+
+void
+intMessageGrp::raw_recv(int sender, void* data, int nbyte)
+{
+  int& seq = source_seq[sender];
+  int msgtype = seq_msgtype(sender,seq);
+#ifdef DEBUG
+  ExEnv::outn() << scprintf("node %d receiving from %d(%d) msgtype = %d\n",
+                   me(),sender,seq,msgtype);
+#endif
+  basic_recv(msgtype, data, nbyte);
+#ifdef DEBUG
+  ExEnv::outn() << scprintf("node %d received %d\n",me(),msgtype);
+#endif
+  if (seq >= seq_mask) seq = 0;
+  else seq++;
+}
+
+void
+intMessageGrp::raw_sendt(int target, int msgtype, const void* data, int nbyte)
+{
+  basic_send(target, typ_msgtype(msgtype), data, nbyte);
+}
+
+void
+intMessageGrp::raw_recvt(int type, void* data, int nbyte)
+{
+  basic_recv(typ_msgtype(type), data, nbyte);
+}
+
+int
+intMessageGrp::probet(int type)
+{
+  return basic_probe(typ_msgtype(type));
+}
+
+int
+intMessageGrp::leftover_ctl_bits()
+{
+  return 3 << ctl_shift;
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/messmpi.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/messmpi.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/messmpi.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,588 @@
+//
+// messmpi.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#include <stdio.h> // for sprintf
+#include <unistd.h> // for fchdir etc.
+#include <fcntl.h> // for open on AIX
+
+#define MPICH_SKIP_MPICXX
+#include <mpi.h>
+extern int MPI_Initialized(int *); // missing in mpi.h
+
+#include <util/keyval/keyval.h>
+#include <util/group/messmpi.h>
+#include <util/misc/formio.h>
+#include <util/misc/newstring.h>
+
+MPI_Comm global_commgrp;
+
+using namespace std;
+using namespace sc;
+
+// Define this to use immediate mode.  This was added added to work
+// around bugs in non-immediate mode optimizations in an MPI impl.
+#undef USE_IMMEDIATE_MODE
+
+// OP_COMMUTES is zero to work around a bug in MPI/Pro 1.5b5 and earlier
+#define OP_COMMUTES 1
+
+///////////////////////////////////////////////////////////////////////
+
+int MPIMessageGrp::nmpi_grps=0;
+Ref<ThreadLock> MPIMessageGrp::grplock;
+
+static
+void
+print_error_and_abort(int me, int mpierror)
+{
+  char msg[MPI_MAX_ERROR_STRING+1];
+  int size;
+  MPI_Error_string(mpierror, msg, &size);
+  msg[size] = '\0';
+  ExEnv::outn() << me << ": " << msg << endl;
+  ExEnv::outn().flush();
+  //MPI_Abort(MPI_COMM_WORLD, mpierror);
+}
+
+static
+const char *
+mpi_thread_string(int level)
+{
+  switch (level) {
+#ifdef HAVE_MPI_INIT_THREAD
+  case MPI_THREAD_SINGLE:
+      return "MPI_THREAD_SINGLE";
+  case MPI_THREAD_FUNNELED:
+      return "MPI_THREAD_FUNNELED";
+  case MPI_THREAD_SERIALIZED:
+      return "MPI_THREAD_SERIALIZED";
+  case MPI_THREAD_MULTIPLE:
+      return "MPI_THREAD_MULTIPLE";
+#endif
+  default:
+      return "unknown";
+    }
+}
+
+///////////////////////////////////////////////////////////////////////
+// The MPIMessageGrp class
+
+static ClassDesc MPIMessageGrp_cd(
+  typeid(MPIMessageGrp),"MPIMessageGrp",1,"public MessageGrp",
+  create<MPIMessageGrp>, create<MPIMessageGrp>, 0);
+
+MPIMessageGrp::MPIMessageGrp()
+{
+  init(MPI_COMM_WORLD);
+}
+
+MPIMessageGrp::MPIMessageGrp(MPI_Comm comm)
+{
+  init(comm);
+}
+
+MPIMessageGrp::MPIMessageGrp(int *argc, char ***argv)
+{
+  init(MPI_COMM_WORLD,argc,argv);
+}
+
+MPIMessageGrp::MPIMessageGrp(const Ref<KeyVal>& keyval):
+  MessageGrp(keyval)
+{
+  if (keyval->exists("argv")) {
+      int argc = keyval->count("argv");
+      char **argv = new char*[argc+1];
+      argv[argc] = 0;
+      for (int arg=0; arg<argc; arg++) {
+          argv[arg] = keyval->pcharvalue("argv",arg);
+        }
+      init(MPI_COMM_WORLD, &argc, &argv);
+    }
+  else {
+      init(MPI_COMM_WORLD);
+    }
+
+  if (keyval->booleanvalue("errors_return")) {
+      if (me()==0)
+          ExEnv::outn() << indent << "MPIMessageGrp: errors_return is true" << endl;
+      MPI_Errhandler_set(commgrp, MPI_ERRORS_RETURN);
+    }
+
+  if (debug_) {
+      ExEnv::outn() << indent << "MPIMessageGrp: KeyVal CTOR: done" << endl;
+    }
+}
+
+void
+MPIMessageGrp::init(MPI_Comm comm, int *argc, char ***argv)
+{
+  int me, nproc;
+
+  if (debug_) {
+      ExEnv::outn() << "MPIMessageGrp::init: entered" << endl;
+    }
+
+  int flag;
+  MPI_Initialized(&flag);
+  if (!flag) {
+      int tmp_argc;
+      char **tmp_argv;
+      int *inits_argc;
+      char ***inits_argv;
+      if (argc && argv) {
+          inits_argc = argc;
+          inits_argv = argv;
+        }
+      else {
+          tmp_argc = 0;
+          tmp_argv = new char*[tmp_argc+1];
+          tmp_argv[tmp_argc] = 0;
+          inits_argc = &tmp_argc;
+          inits_argv = &tmp_argv;
+        }
+      // This dot business is to work around problems with some MPI
+      // implementations.
+      int dot = open(".",O_RDONLY);
+      if (debug_) {
+          ExEnv::outn() << indent
+               << "Calling MPI_Init with";
+          for (int i=0; i<*argc; i++) {
+              ExEnv::outn() << " " << *argv[i];
+            }
+          ExEnv::outn() << endl;
+        }
+#ifdef HAVE_MPI_INIT_THREAD
+      int provided, desired = SC_MPI_THREAD_LEVEL;
+      MPI_Init_thread(inits_argc, inits_argv, desired, &provided);
+      int me;
+      MPI_Comm_rank(MPI_COMM_WORLD, &me);
+      if (provided != desired && me == 0) {
+          ExEnv::outn() << indent
+                        << "WARNING: desired "
+                        << mpi_thread_string(desired)
+                        << " MPI threading support but got "
+                        << mpi_thread_string(provided)
+                        << endl;
+        }
+#else
+      MPI_Init(inits_argc, inits_argv);
+#endif
+#ifdef HAVE_FCHDIR
+      fchdir(dot);
+#endif
+      close(dot);
+    }
+
+  MPI_Comm_dup(comm, &commgrp);
+  global_commgrp = commgrp;
+
+  MPI_Errhandler_set(commgrp, MPI_ERRORS_ARE_FATAL);
+
+   if (!nmpi_grps) {
+      threadgrp = ThreadGrp::get_default_threadgrp();
+      grplock = threadgrp->new_lock();
+     }
+      grplock->lock();
+      nmpi_grps++;
+      grplock->unlock();
+
+  MPI_Comm_rank(commgrp,&me);
+  MPI_Comm_size(commgrp, &nproc);
+  bufsize = 4000000;
+  buf = 0;
+  //buf = (void*) new char[bufsize];
+  //MPI_Buffer_attach(buf,bufsize);
+
+  if (getenv("MPIMESSAGEGRP_MESSAGEGRP_COLLECTIVES"))
+      use_messagegrp_collectives_ = true;
+  else
+      use_messagegrp_collectives_ = false;
+  
+  initialize(me, nproc);
+
+  //MPIL_Trace_on();
+
+  if (debug_) {
+      ExEnv::outn() << me << ": MPIMessageGrp::init: done" << endl;
+    }
+
+  SCFormIO::init_mp(me);
+}
+
+MPIMessageGrp::~MPIMessageGrp()
+{
+  //MPIL_Trace_off();
+  //MPI_Buffer_detach(&buf, &bufsize);
+  delete[] (char*) buf;
+
+  grplock->lock();
+  nmpi_grps--;
+  if (!nmpi_grps) MPI_Finalize();
+  grplock->unlock();
+  
+}
+
+Ref<MessageGrp> MPIMessageGrp::clone(void)
+{
+  Ref<MessageGrp> mgrp = new MPIMessageGrp;
+  return mgrp;
+}
+
+void
+MPIMessageGrp::raw_send(int target, const void* data, int nbyte)
+{
+  if (debug_) {
+      ExEnv::outn() << scprintf("%3d: MPI_Send"
+                       "(0x%08x, %5d, MPI_BYTE, %3d, 0, commgrp)",
+                       me(), data, nbyte, target)
+           << endl;
+    }
+  int ret;
+#ifndef USE_IMMEDIATE_MODE
+  ret = MPI_Send(const_cast<void*>(data),nbyte,MPI_BYTE,target,0,commgrp);
+#else
+  MPI_Request mpireq;
+  MPI_Status status;
+  ret = MPI_Isend(data,nbyte,MPI_BYTE,target,0,commgrp,&mpireq);
+  if (ret == MPI_SUCCESS) ret = MPI_Wait(&mpireq,&status);
+#endif // USE_IMMEDIATE_MODE
+  if (ret != MPI_SUCCESS) {
+      ExEnv::outn() << me() << ": MPIMessageGrp::raw_send("
+          << target << ",," << nbyte << "): mpi error:" << endl;
+      print_error_and_abort(me(), ret);
+    }
+  if (debug_) ExEnv::outn() << scprintf("%3d: sent\n", me()) << endl;
+}
+
+void
+MPIMessageGrp::raw_recv(int sender, void* data, int nbyte)
+{
+  MPI_Status status;
+  if (sender == -1) sender = MPI_ANY_SOURCE;
+  if (debug_) {
+      ExEnv::outn() << scprintf("%3d: MPI_Recv"
+                       "(0x%08x, %5d, MPI_BYTE, %3d, 0, commgrp,)",
+                       me(), data, nbyte, sender)
+           << endl;
+    }
+  int ret;
+#ifndef USE_IMMEDIATE_MODE
+  ret = MPI_Recv(data,nbyte,MPI_BYTE,sender,0,commgrp,&status);
+#else
+  MPI_Request mpireq;
+  ret = MPI_Irecv(data,nbyte,MPI_BYTE,sender,0,commgrp,&mpireq);
+  if (ret == MPI_SUCCESS) ret = MPI_Wait(&mpireq,&status);
+#endif // USE_IMMEDIATE_MODE
+  if (ret != MPI_SUCCESS) {
+      ExEnv::outn() << me() << ": MPIMessageGrp::raw_recv("
+          << sender << ",," << nbyte << "): mpi error:" << endl;
+      print_error_and_abort(me(), ret);
+    }
+  rnode = status.MPI_SOURCE;
+  rtag = status.MPI_TAG;
+  rlen = nbyte;
+  if (debug_) ExEnv::outn() << scprintf("%3d: recvd %d bytes\n", me(), rlen) << endl;
+}
+
+void
+MPIMessageGrp::raw_sendt(int target, int type, const void* data, int nbyte)
+{
+  type = (type<<1) + 1;
+  if (debug_) {
+      ExEnv::outn() << scprintf("%3d: MPI_Send"
+                       "(0x%08x, %5d, MPI_BYTE, %3d, %5d, commgrp)",
+                       me(), data, nbyte, target, type)
+           << endl;
+    }
+  int ret;
+#ifndef USE_IMMEDIATE_MODE
+  ret = MPI_Send(const_cast<void*>(data),nbyte,MPI_BYTE,target,type,commgrp);
+#else
+  MPI_Request mpireq;
+  MPI_Status status;
+  ret = MPI_Isend(data,nbyte,MPI_BYTE,target,type,commgrp,&mpireq);
+  if (ret == MPI_SUCCESS) ret = MPI_Wait(&mpireq,&status);
+#endif
+  if (ret != MPI_SUCCESS) {
+      ExEnv::outn() << me() << ": MPIMessageGrp::raw_sendt("
+          << target << "," << type << ",," << nbyte << "): mpi error:" << endl;
+      print_error_and_abort(me(), ret);
+    }
+  if (debug_) ExEnv::outn() << scprintf("%3d: sent\n", me()) << endl;
+}
+
+void
+MPIMessageGrp::raw_recvt(int type, void* data, int nbyte)
+{
+  MPI_Status status;
+  if (type == -1) type = MPI_ANY_TAG;
+  else type = (type<<1) + 1;
+  if (debug_) {
+      ExEnv::outn() << scprintf("%3d: MPI_Recv(0x%08x, %5d, MPI_BYTE, "
+                       "MPI_ANY_SOURCE, %5d, commgrp,)",
+                       me(), data, nbyte, type)
+           << endl;
+    }
+  int ret;
+#ifndef USE_IMMEDIATE_MODE
+  ret = MPI_Recv(data,nbyte,MPI_BYTE,MPI_ANY_SOURCE,type,commgrp,&status);
+#else
+  MPI_Request mpireq;
+  ret = MPI_Irecv(data,nbyte,MPI_BYTE,MPI_ANY_SOURCE,type,commgrp,&mpireq);
+  if (ret == MPI_SUCCESS) ret = MPI_Wait(&mpireq,&status);
+#endif // USE_IMMEDIATE_MODE
+  if (ret != MPI_SUCCESS) {
+      ExEnv::outn() << me() << ": MPIMessageGrp::raw_recvt("
+          << type << ",," << nbyte << "): mpi error:" << endl;
+      print_error_and_abort(me(), ret);
+    }
+  rnode = status.MPI_SOURCE;
+  rtag = status.MPI_TAG;
+  rlen = nbyte;
+  if (debug_) {
+      ExEnv::outn() << scprintf("%3d: recvd %d bytes from %d with tag %d\n",
+                       me(), rlen, rnode, rtag) << endl;
+    }
+}
+
+int
+MPIMessageGrp::probet(int type)
+{
+  int flag;
+  MPI_Status status;
+
+  if (type == -1) type = MPI_ANY_TAG;
+  else type = (type<<1) + 1;
+  int ret;
+  if (debug_) {
+      ExEnv::outn() << scprintf("%3d: MPI_Iprobe(MPI_ANY_SOURCE, %5d, commgrp, "
+                       "&flag, &status)", me(), type)
+           << endl;
+    }
+  if ((ret = MPI_Iprobe(MPI_ANY_SOURCE,type,commgrp,&flag,&status))
+      != MPI_SUCCESS ) {
+      ExEnv::outn() << me() << ": MPIMessageGrp::probet("
+          << type << "): mpi error:" << endl;
+      print_error_and_abort(me(), ret);
+    }
+  if (flag) {
+    rnode = status.MPI_SOURCE;
+    rtag = status.MPI_TAG;
+    MPI_Get_count(&status, MPI_BYTE, &rlen);
+    return 1;
+    }
+  else {
+    rnode = rtag = rlen = 0;
+    }
+    
+  return 0;
+}
+
+void
+MPIMessageGrp::sync()
+{
+  int ret;
+  if (debug_) {
+      ExEnv::outn() << scprintf("%3d: MPI_Barrier(commgrp)", me()) << endl;
+    }
+  if ((ret = MPI_Barrier(commgrp)) != MPI_SUCCESS) {
+      ExEnv::outn() << me() << ": MPIMessageGrp::sync(): mpi error:" << endl;
+      print_error_and_abort(me(), ret);
+    }
+}
+
+#define REDUCEMEMBER(name, type, mpitype) \
+static GrpReduce<type>* name ## reduceobject; \
+extern "C" void \
+name ## reduce(void*b, void*a, int*len, MPI_Datatype*datatype) \
+{ \
+  name ## reduceobject->reduce((type*)a, (type*)b, *len); \
+} \
+void \
+MPIMessageGrp::reduce(type*d, int n, GrpReduce<type>&r, \
+                      type*scratch, int target) \
+{ \
+  if (use_messagegrp_collectives_) { \
+      MessageGrp::reduce(d,n,r,scratch,target); \
+      return; \
+    } \
+ \
+  name ## reduceobject = &r; \
+ \
+  MPI_Op op; \
+  MPI_Op_create(name ## reduce, OP_COMMUTES, &op); \
+ \
+  type *work; \
+  if (!scratch) work = new type[n]; \
+  else work = scratch; \
+ \
+  int ret; \
+ \
+  if (target == -1) { \
+      if (debug_) { \
+          ExEnv::outn() << scprintf("%3d: MPI_Allreduce" \
+          "(0x%08x, 0x%08x, %5d, %3d, op, commgrp)", \
+          me(), d, work, n, mpitype) \
+               << endl; \
+        } \
+      ret = MPI_Allreduce(d, work, n, mpitype, op, commgrp); \
+      if (debug_) \
+        ExEnv::outn() << scprintf("%3d: done with Allreduce", me()) << endl; \
+    } \
+  else { \
+      if (debug_) { \
+          ExEnv::outn() << scprintf("%3d: MPI_Reduce" \
+          "(0x%08x, 0x%08x, %5d, %3d, op, %3d, commgrp)", \
+          me(), d, work, n, mpitype, target) \
+               << endl; \
+        } \
+      ret = MPI_Reduce(d, work, n, mpitype, op, target, commgrp); \
+      if (debug_) \
+        ExEnv::outn() << scprintf("%3d: done with Reduce", me()) << endl; \
+    } \
+ \
+  if (ret != MPI_SUCCESS) { \
+      ExEnv::outn() << me() << ": MPIMessageGrp::reduce(," \
+          << n << ",,," << target << "): mpi error:" << endl; \
+      print_error_and_abort(me(), ret); \
+    } \
+ \
+  if (target == -1 || target == me()) { \
+     for (int i=0; i<n; i++) d[i] = work[i]; \
+    } \
+ \
+  MPI_Op_free(&op); \
+ \
+  if (!scratch) delete[] work; \
+}
+
+REDUCEMEMBER(double, double, MPI_DOUBLE)
+REDUCEMEMBER(float, float, MPI_FLOAT)
+REDUCEMEMBER(uint, unsigned int, MPI_INT)
+REDUCEMEMBER(int, int, MPI_INT)
+REDUCEMEMBER(short, short, MPI_SHORT)
+REDUCEMEMBER(long, long, MPI_LONG)
+REDUCEMEMBER(char, char, MPI_CHAR)
+REDUCEMEMBER(uchar, unsigned char, MPI_UNSIGNED_CHAR)
+#ifdef MPI_SIGNED_CHAR
+REDUCEMEMBER(schar, signed char, MPI_SIGNED_CHAR)
+#else
+void
+MPIMessageGrp::reduce(signed char* d, int n, GrpReduce<signed char>& r,
+                      signed char*scratch, int target)
+{
+  MessageGrp::reduce(d,n,r,scratch,target);
+}
+#endif
+
+#define SUMMEMBER(name, type, mpitype) \
+void \
+MPIMessageGrp::sum(type*d, int n, type*scratch, int target) \
+{ \
+  if (use_messagegrp_collectives_) { \
+      MessageGrp::sum(d,n,scratch,target); \
+      return; \
+    } \
+ \
+  type *work; \
+  if (!scratch) work = new type[n]; \
+  else work = scratch; \
+ \
+  int ret; \
+ \
+  if (target == -1) { \
+      if (debug_) { \
+          ExEnv::outn() << scprintf("%3d: MPI_Allreduce" \
+          "(0x%08x, 0x%08x, %5d, %3d, MPI_SUM, commgrp)", \
+          me(), d, work, n, mpitype) \
+               << endl; \
+        } \
+      ret = MPI_Allreduce(d, work, n, mpitype, MPI_SUM, commgrp); \
+      if (debug_) \
+        ExEnv::outn() << scprintf("%3d: done with Allreduce", me()) << endl; \
+    } \
+  else { \
+      if (debug_) { \
+          ExEnv::outn() << scprintf("%3d: MPI_Reduce" \
+          "(0x%08x, 0x%08x, %5d, %3d, MPI_SUM, %3d, commgrp)", \
+          me(), d, work, n, mpitype, target) \
+               << endl; \
+        } \
+      ret = MPI_Reduce(d, work, n, mpitype, MPI_SUM, target, commgrp); \
+      if (debug_) \
+        ExEnv::outn() << scprintf("%3d: done with Reduce", me()) << endl; \
+    } \
+ \
+  if (ret != MPI_SUCCESS) { \
+      ExEnv::outn() << me() << ": MPIMessageGrp::sum(," \
+          << n << ",,," << target << "): mpi error:" << endl; \
+      print_error_and_abort(me(), ret); \
+    } \
+ \
+  if (target == -1 || target == me()) { \
+     for (int i=0; i<n; i++) d[i] = work[i]; \
+    } \
+ \
+  if (!scratch) delete[] work; \
+}
+SUMMEMBER(double, double, MPI_DOUBLE)
+SUMMEMBER(int, int, MPI_INT)
+
+void
+MPIMessageGrp::raw_bcast(void* data, int nbyte, int from)
+{
+  if (n() == 1) return;
+
+  if (use_messagegrp_collectives_) {
+      MessageGrp::raw_bcast(data,nbyte,from);
+      return;
+    }
+
+  if (debug_) {
+      ExEnv::outn() << scprintf("%3d: MPI_Bcast("
+                       "0x%08x, %5d, MPI_BYTE, %3d, commgrp)",
+                       me(), data, nbyte, from)
+           << endl;
+    }
+  int ret;
+  if ((ret = MPI_Bcast(data, nbyte, MPI_BYTE, from, commgrp))
+      != MPI_SUCCESS) {
+      ExEnv::outn() << me() << ": MPIMessageGrp::raw_bcast(,"
+          << nbyte << "," << from << "): mpi error:" << endl;
+      print_error_and_abort(me(), ret);
+    }
+  if (debug_) {
+      ExEnv::outn() << scprintf("%3d: done with bcast", me()) << endl;
+    }
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/messmpi.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/messmpi.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/messmpi.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,121 @@
+//
+// messmpi.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_messmpi_h
+#define _util_group_messmpi_h
+
+#include <util/group/message.h>
+#include <util/group/thread.h>
+
+#define MPICH_SKIP_MPICXX
+#include <mpi.h>
+
+namespace sc {
+
+/** The MPIMessageGrp class is an concrete implementation of MessageGrp
+that uses the MPI 1 library.  */
+class MPIMessageGrp: public MessageGrp {
+  protected:
+    void* buf;
+    int bufsize;
+
+    int rnode;
+    int rtag;
+    int rlen;
+
+    /// If true use the generic collective routines in the base class
+    bool use_messagegrp_collectives_;
+
+    /// Number of MPIMessageGrp's currently in use.
+    static int nmpi_grps;
+    /// lock to access nmpi_grps variable
+    static Ref<ThreadLock> grplock;
+
+    Ref<ThreadGrp> threadgrp;
+    /// Currently each commgrp is a dup of MPI_COMM_WORLD
+    MPI_Comm commgrp;
+    
+    /// Not thread-safe due to race condition on nmpi_grps variable.
+    void init(MPI_Comm comm, int *argc=0, char ***argv=0);
+  public:
+    MPIMessageGrp();
+    /** Use an MPI communicator to create a MessageGrp.  The comm
+        argument could be a subset of MPI_COMM_WORLD, for example. */
+    MPIMessageGrp(MPI_Comm comm);
+    /** Use argc and argv to create a MPIMessageGrp.  This would
+        have to be used for implementations of MPI that have MPI_Init
+        fill in argc and argv. */
+    MPIMessageGrp(int *argc, char ***argv);
+    /** Construction MPIMessageGrp given a KeyVal input object. */
+    MPIMessageGrp(const Ref<KeyVal>&);
+    ~MPIMessageGrp();
+
+    /// Clones (dups) an MPIMessageGrp from MPI_COMM_WORLD 
+    Ref<MessageGrp> clone(void);
+    
+    void raw_send(int target, const void* data, int nbyte);
+    void raw_recv(int sender, void* data, int nbyte);
+    void raw_sendt(int target, int type, const void* data, int nbyte);
+    void raw_recvt(int type, void* data, int nbyte);
+
+    int probet(int type);
+
+    void sync();
+
+    void sum(double*, int n, double*scratch = 0, int target = -1);
+    void sum(int*, int n, int*scratch = 0, int target = -1);
+
+    void reduce(double*, int n, GrpReduce<double>&,
+                double*scratch = 0, int target = -1);
+    void reduce(unsigned int*, int n, GrpReduce<unsigned int>&,
+                unsigned int*scratch = 0, int target = -1);
+    void reduce(int*, int n, GrpReduce<int>&,
+                int*scratch = 0, int target = -1);
+    void reduce(char*, int n, GrpReduce<char>&,
+                char*scratch = 0, int target = -1);
+    void reduce(unsigned char*, int n, GrpReduce<unsigned char>&,
+                unsigned char*scratch = 0, int target = -1);
+    void reduce(signed char*, int n, GrpReduce<signed char>&,
+                signed char*scratch = 0, int target = -1);
+    void reduce(short*, int n, GrpReduce<short>&,
+                short*scratch = 0, int target = -1);
+    void reduce(float*, int n, GrpReduce<float>&,
+                float*scratch = 0, int target = -1);
+    void reduce(long*, int n, GrpReduce<long>&,
+                long*scratch = 0, int target = -1);
+
+    void raw_bcast(void* data, int nbyte, int from);
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/messproc.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/messproc.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/messproc.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,188 @@
+//
+// messproc.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#include <util/misc/formio.h>
+#include <util/group/message.h>
+
+using namespace sc;
+
+static ClassDesc ProcMessageGrp_cd(
+  typeid(ProcMessageGrp),"ProcMessageGrp",1,"public MessageGrp",
+  0, create<ProcMessageGrp>, 0);
+
+ProcMessageGrp::ProcMessageGrp(const Ref<KeyVal>& keyval):
+  MessageGrp(keyval)
+{
+  sync_messages=0;
+  type_messages=0;
+  initialize(0,1);
+}
+
+ProcMessageGrp::ProcMessageGrp()
+{
+  sync_messages=0;
+  type_messages=0;
+  initialize(0,1);
+}
+
+ProcMessageGrp::~ProcMessageGrp()
+{
+}
+
+Ref<MessageGrp> ProcMessageGrp::clone(void)
+{
+  Ref<MessageGrp> pmg = new ProcMessageGrp;
+  return pmg;
+}
+
+void ProcMessageGrp::sendit(message_t *& messages, int dest, int msgtype, const void* buf,
+                            int bytes)
+{
+  message_t *msg;
+  message_t *I;
+
+  if (dest != 0) {
+      ExEnv::errn() << scprintf("messproc.cc:sendit: can only send to 0\n");
+      abort();
+    }
+
+  msg = (message_t *) malloc(sizeof(message_t));
+  if (msg) msg->buf = (char *) malloc(bytes);
+  if (!msg || !msg->buf) {
+      ExEnv::errn() << scprintf("messproc.cc:sendit: allocation failed\n");
+      abort();
+    }
+
+  // Put msg at the end of the linked list, because of some bad
+  // assumptions made by the mpscf program and libraries.
+  msg->p = 0;
+  if (!messages) {
+      messages = msg;
+    }
+  else {
+      for (I=messages; I->p != 0; I=I->p);
+      I->p = msg;
+    }
+
+  memcpy(msg->buf,buf,bytes);
+  msg->type = msgtype;
+  msg->size = bytes;
+}
+
+void ProcMessageGrp::recvit(message_t *& messages, int source, int type, void* buf,
+                            int bytes, int& last_size, int& last_type)
+{
+  message_t *i;
+  message_t *last;
+
+  last = 0;
+  for (i=messages; i!=0; i = i->p) {
+    if (i->type == type || type == -1) {
+      if (i->size > bytes) {
+        ExEnv::errn() << scprintf(
+                "messproc.cc:recvit: message buffer isn't big enough\n");
+        abort();
+        }
+      memcpy(buf,i->buf,i->size);
+
+      // Remove the message from the list.
+      if (last) {
+          last->p = i->p;
+        }
+      else {
+          messages = messages->p;
+        }
+      free(i->buf);
+      free(i);
+
+      return;
+      }
+    last = i;
+    }
+
+  ExEnv::errn() << scprintf(
+          "messproc.cc:recvit: tried to receive something that isn't there\n");
+  ExEnv::errn() << scprintf("messproc:recvit: tried %d bytes of type %d, ",bytes,type);
+  abort();
+}
+
+void
+ProcMessageGrp::raw_send(int target, const void* data, int nbyte)
+{
+  sendit(sync_messages, target, -1, data, nbyte);
+}
+
+void
+ProcMessageGrp::raw_sendt(int target, int type, const void* data, int nbyte)
+{
+  sendit(type_messages, target, type, data, nbyte);
+}
+
+void
+ProcMessageGrp::raw_recv(int sender, void* data, int nbyte)
+{
+  int last_size, last_type;
+  recvit(sync_messages, sender, -1, data, nbyte, last_size, last_type);
+}
+
+void
+ProcMessageGrp::raw_recvt(int type, void* data, int nbyte)
+{
+  int last_size, last_type;
+  recvit(type_messages, -1, type, data, nbyte, last_size, last_type);
+}
+
+void
+ProcMessageGrp::raw_bcast(void* data, int nbyte, int from)
+{
+}
+
+int
+ProcMessageGrp::probet(int type)
+{
+  message_t *i;
+
+  for (i=type_messages; i!=0; i = i->p) {
+      if (i->type == type || type == -1) {
+          return 1;
+        }
+    }
+
+  return 0;
+}
+
+void
+ProcMessageGrp::sync()
+{
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/messshm.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/messshm.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/messshm.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,466 @@
+//
+// messshm.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/ipc.h>
+#include <sys/sem.h>
+#include <sys/shm.h>
+
+
+#include <util/misc/bug.h>
+#include <util/misc/formio.h>
+#include <util/group/messshm.h>
+
+using namespace std;
+using namespace sc;
+
+//#define DEBUG
+
+#ifndef SEM_A
+#  define SEM_A 0200
+#endif
+
+#ifndef SEM_R
+#  define SEM_R 0400
+#endif
+
+/* NALIGN is the byte boundary that we align data on. */
+#define NALIGN 8
+#define ROUNDUPTOALIGN(n) (((n) + (NALIGN-1)) & ~(NALIGN-1))
+
+static ClassDesc ShmMessageGrp_cd(
+  typeid(ShmMessageGrp),"ShmMessageGrp",1,"public intMessageGrp",
+  0, create<ShmMessageGrp>, 0);
+
+ShmMessageGrp::ShmMessageGrp()
+{
+  initialize();
+}
+
+ShmMessageGrp::ShmMessageGrp(int nprocs)
+{
+  initialize(nprocs);
+}
+
+ShmMessageGrp::ShmMessageGrp(const Ref<KeyVal>& keyval):
+  intMessageGrp(keyval)
+{
+  int nprocs = keyval->intvalue("n");
+  if (keyval->error() != KeyVal::OK) initialize();
+  else initialize(nprocs);
+}
+
+void ShmMessageGrp::sync()
+{
+  int i;
+  for (i=0; i<n(); i++) {
+      if (me() == i) continue;
+      wait_for_write(i);
+      commbuf[i]->n_sync++;
+      if (commbuf[i]->n_sync >= n()-1) {
+          while(commbuf[i]->n_wait_for_change) {
+              put_change(i);
+              commbuf[i]->n_wait_for_change--;
+            }
+        }
+      release_write(i);
+    }
+  wait_for_write(me());
+  while (commbuf[me()]->n_sync < n()-1) {
+      commbuf[me()]->n_wait_for_change++;
+      release_write(me());
+      get_change(me());
+      wait_for_write(me());
+    }
+  commbuf[me()]->n_sync -= n()-1;
+  while(commbuf[me()]->n_wait_for_change) {
+      put_change(me());
+      commbuf[me()]->n_wait_for_change--;
+    }
+  release_write(me());
+}
+
+ShmMessageGrp::~ShmMessageGrp()
+{
+  // sync the nodes
+  sync();
+
+  // make sure node zero is las to touch the shared memory
+  if (me() == 0) {
+      wait_for_write(0);
+      while (commbuf[0]->n_sync < n()-1) {
+          commbuf[0]->n_wait_for_change++;
+          release_write(0);
+          get_change(0);
+          wait_for_write(0);
+        }
+      release_write(0);
+      shmdt((SHMTYPE)sharedmem);
+      // release the memory
+      shmctl(shmid,IPC_RMID,0);
+
+      for (int i=0; i<n(); i++) {
+#ifdef SEMCTL_REQUIRES_SEMUN
+          semun junk;
+          junk.val = 0;
+#else
+          int junk = 0;
+#endif
+          semctl(semid,i,IPC_RMID,junk);
+          semctl(change_semid,i,IPC_RMID,junk);
+        }
+    }
+  else {
+      wait_for_write(0);
+      commbuf[0]->n_sync++;
+      while(commbuf[0]->n_wait_for_change) {
+          put_change(0);
+          commbuf[0]->n_wait_for_change--;
+        }
+      shmdt((SHMTYPE)sharedmem);
+      release_write(0);
+    }
+}
+
+void ShmMessageGrp::initialize()
+{
+  int nprocs = atoi(getenv("NUMPROC"));
+  if (!nprocs) nprocs = 1;
+  initialize(nprocs);
+}
+
+void ShmMessageGrp::initialize(int nprocs)
+{
+  int i;
+  void* nextbuf;
+
+  semdec.sem_num = 0;
+  semdec.sem_op = -1;
+  semdec.sem_flg = 0;
+  seminc.sem_num = 0;
+  seminc.sem_op =  1;
+  seminc.sem_flg = 0;
+
+  // Each node gets a buffer for incoming data.
+  shmid = shmget(IPC_PRIVATE,
+                 nprocs * sizeof(commbuf_t),
+                 IPC_CREAT | SHM_R | SHM_W);
+
+  // Attach the shared segment.
+  nextbuf = sharedmem = shmat(shmid,0,0);
+
+#ifdef SEMCTL_REQUIRES_SEMUN
+  semun semzero;
+  semzero.val = 0;
+  semun semone;
+  semone.val = 1;
+#else
+  int semzero = 0;
+  int semone = 1;
+#endif
+
+  // Each shared memory segment gets a semaphore to synchronize access.
+  semid = semget(IPC_PRIVATE,nprocs,IPC_CREAT | SEM_R | SEM_A );
+  if (semid == -1) {
+      perror("semget");
+      exit(-1);
+    }
+  
+  change_semid = semget(IPC_PRIVATE,nprocs,IPC_CREAT | SEM_R | SEM_A );
+  if (change_semid == -1) {
+      perror("semget");
+      exit(-1);
+    }
+
+  for (i=0; i<nprocs; i++) {
+
+      // Mark all of the segments as available for writing.
+      if (semctl(semid,i,SETVAL,semone) == -1) {
+          perror("semctl");
+          exit(-1);
+        }
+
+      if (semctl(change_semid,i,SETVAL,semzero) == -1) {
+          perror("semctl");
+          exit(-1);
+        }
+
+      // Alloc shm for each node's commbuf.
+      commbuf[i] = (commbuf_t*) nextbuf;
+      // Mark the commbuf as having no messages.
+      commbuf[i]->nmsg = 0;
+      // and no outstanding waits
+      commbuf[i]->n_wait_for_change = 0;
+      commbuf[i]->n_sync = 0;
+      nextbuf = (void*)(((char*)nextbuf) + sizeof(commbuf_t));
+    }
+
+  // Create the remaining nodes.
+  int mynodeid = 0;
+  for (i=1; i<nprocs; i++) {
+      int pid = fork();
+      if (!pid) {
+          mynodeid = i;
+          break;
+        }
+    }
+
+  // Initialize the base class.
+  intMessageGrp::initialize(mynodeid, nprocs, 30);
+}
+
+Ref<MessageGrp>
+ShmMessageGrp::clone(void)
+{
+  Ref<MessageGrp> smgrp = new ShmMessageGrp(n());
+  return smgrp;
+}
+
+int ShmMessageGrp::basic_probe(int msgtype)
+{
+  int i;
+  msgbuf_t *message;
+
+  wait_for_write(me());
+
+  message = (msgbuf_t*)commbuf[me()]->buf;
+  for (i=0; i<commbuf[me()]->nmsg; i++) {
+      if ((msgtype == -1) || (msgtype == message->type)) {
+          release_write(me());
+          return 1;
+        }
+      message = NEXT_MESSAGE(message);
+    }
+
+  release_write(me());
+
+  return 0;
+}
+
+void ShmMessageGrp::basic_recv(int type, void* buf, int bytes)
+{
+  int size;
+  int i;
+  msgbuf_t *message,*lastmessage;
+
+#ifdef DEBUG
+  ExEnv::outn() << "ShmGrp: basic_recv: "
+       << "type = " << type << ' '
+       << "buf = " << buf << ' '
+       << "bytes = " << bytes << ' '
+       << "me = " << me() << endl;
+  print_buffer(me(),me());
+#endif
+
+  look_for_message:
+
+  wait_for_write(me());
+
+  message = (msgbuf_t*)commbuf[me()]->buf;
+  for (i=0; i<commbuf[me()]->nmsg; i++) {
+      if (message->type == type) break;
+      message = NEXT_MESSAGE(message);
+    }
+  if (i==commbuf[me()]->nmsg) {
+      commbuf[me()]->n_wait_for_change++;
+      release_write(me());
+      get_change(me());
+      goto look_for_message;
+    }
+
+  if (bytes < message->size) {
+      ExEnv::errn() << scprintf("messshm.cc: recv buffer too small\n");
+      abort();
+    }
+  if (bytes < message->size) size = bytes;
+  else size = message->size;
+
+  // Copy the data.
+  memcpy(buf,((char*)message) + sizeof(msgbuf_t),size);
+
+  // Find the lastmessage.
+  lastmessage = (msgbuf_t*)commbuf[me()]->buf;
+  for (i=0; i<commbuf[me()]->nmsg; i++) {
+      lastmessage = NEXT_MESSAGE(lastmessage);
+    }
+
+  // Repack the message buffer.
+  memmove(message,NEXT_MESSAGE(message),
+          (size_t)((char*)lastmessage)-(size_t)((char*)NEXT_MESSAGE(message)));
+
+  commbuf[me()]->nmsg--;
+
+  while(commbuf[me()]->n_wait_for_change) {
+      put_change(me());
+      commbuf[me()]->n_wait_for_change--;
+    }
+
+  release_write(me());
+}
+
+void ShmMessageGrp::basic_send(int dest, int type, const void* buf, int bytes)
+{
+  int i;
+  msgbuf_t *availmsg;
+
+#ifdef DEBUG
+  ExEnv::outn() << "ShmGrp: basic_send: "
+       << "dest = " << dest << ' '
+       << "type = " << type << ' '
+       << "buf = " << buf << ' '
+       << "bytes = " << bytes << ' '
+       << "me = " << me() << endl;
+#endif
+
+  if (dest>=n()) {
+      //debug_start("bad destination");
+      ExEnv::errn() << scprintf("ShmMessageGrp::basic_send: bad destination\n");
+      abort();
+    }
+
+  try_send_again:
+
+  // Obtain write access to the dest's incoming buffer.
+  wait_for_write(dest);
+
+  availmsg = (msgbuf_t*)commbuf[dest]->buf;
+  for (i=0; i<commbuf[dest]->nmsg; i++) {
+      availmsg = NEXT_MESSAGE(availmsg);
+    }
+  if (  (((char*)availmsg) + ROUNDUPTOALIGN(sizeof(msgbuf_t) + bytes))
+        > (((char*)commbuf[dest]) + sizeof(commbuf_t))) {
+      if (me() == dest) {
+          // sending a message to myself and the buffer is full
+          // --cannot recover
+          ExEnv::errn() << scprintf("commbuf size exceeded on %d\n",me());
+          ExEnv::errn() << scprintf(" availmsg = 0x%x\n",availmsg);
+          ExEnv::errn() << scprintf(" commbuf[%d] + sizeof(commbuf_t) = 0x%x\n",
+                           dest,((char*)commbuf[dest]) + sizeof(commbuf_t));
+          ExEnv::errn() << scprintf(" size = %d\n",bytes);
+          abort();
+        }
+      else {
+          // try to recover from a full buffer by waiting for the dest
+          // to read some data.
+          commbuf[dest]->n_wait_for_change++;
+          release_write(dest);
+          get_change(dest);
+          goto try_send_again;
+        }
+    }
+  availmsg->from = me();
+  availmsg->type = type;
+  availmsg->size = bytes;
+  memcpy(((char*)availmsg) + sizeof(msgbuf_t),buf,bytes);
+  commbuf[dest]->nmsg++;
+
+  // let the dest know that there is more data in the buffer
+  while(commbuf[dest]->n_wait_for_change) {
+      put_change(dest);
+      commbuf[dest]->n_wait_for_change--;
+    }
+
+  // Release write access to the dest's buffer.
+  release_write(dest);
+}
+
+msgbuf_t * ShmMessageGrp::NEXT_MESSAGE(msgbuf_t *m)
+{
+  msgbuf_t *r;
+  if (m->size < 0) {
+      ExEnv::errn() << scprintf("NEXT_MESSAGE: m->size = %d (real %d)\n",
+                       m->size,sizeof(msgbuf_t) + m->size + m->size%8);
+      //debug_start("m->size < 0");
+      ExEnv::errn() << scprintf("messshm.cc: m->size < 0\n");
+      abort();
+    }
+  r = ((msgbuf_t*)(((char*)m) + ROUNDUPTOALIGN(sizeof(msgbuf_t) + m->size)));
+  return r;
+}
+
+void ShmMessageGrp::get_change(int node)
+{
+  semdec.sem_num = node;
+  semop(change_semid,&semdec,1);
+  semdec.sem_num = 0;
+}
+
+void ShmMessageGrp::put_change(int node)
+{
+  seminc.sem_num = node;
+  semop(change_semid,&seminc,1);
+  seminc.sem_num = 0;  
+
+}
+
+// Obtain a lock for writing to the node's buffer.
+void ShmMessageGrp::wait_for_write(int node)
+{
+  semdec.sem_num = node;
+  semop(semid,&semdec,1);
+  semdec.sem_num = 0;
+}
+
+// Release lock for writing to node's buffer.
+void ShmMessageGrp::release_write(int node)
+{
+  seminc.sem_num = node;
+  semop(semid,&seminc,1);
+  seminc.sem_num = 0;
+}
+
+#ifdef DEBUG
+void ShmMessageGrp::print_buffer(int node, int me)
+{
+  int i;
+  msgbuf_t *message;
+  message = (msgbuf_t*)commbuf[node]->buf;
+
+  ExEnv::outn() << scprintf("Printing buffer for node %d on node %d\n",node,me);
+  for (i=0; i<commbuf[node]->nmsg; i++) {
+      ExEnv::outn() <<
+          scprintf(" on node %2d: to=%2d, bytes=%6d, type=%10d, from=%2d\n",
+                   me,
+                   node,
+                   message->size,
+                   message->type,
+                   message->from);
+      ExEnv::outn().flush();
+      message = NEXT_MESSAGE(message);
+    }
+
+}
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/messshm.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/messshm.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/messshm.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,122 @@
+//
+// messshm.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_messshm_h
+#define _util_group_messshm_h
+
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/ipc.h>
+#include <sys/sem.h>
+#include <sys/shm.h>
+
+#include <util/group/message.h>
+
+namespace sc {
+
+#define SHMCOMMBUFSIZE 1500000
+
+/* Set the maximum number of processors (including the host). */
+#define MAXPROCS 17
+
+
+struct commbuf_struct {
+    int nmsg;
+    int n_wait_for_change;
+    int n_sync;
+    char buf[SHMCOMMBUFSIZE];
+};
+typedef struct commbuf_struct commbuf_t;
+
+struct msgbuf_struct {
+    int type;
+    int from;
+    int size;
+};
+typedef struct msgbuf_struct msgbuf_t;
+
+/** The ShmMessageGrp class is an implementation of MessageGrp that
+allows multiple process to be started that communicate with shared memory.
+This only provides improved performance if you have multiple CPU's in a
+symmetric multiprocessor configuration.  Nonetheless, it is quite useful on
+a single CPU for tracking down bugs.
+*/
+class ShmMessageGrp: public intMessageGrp {
+  protected:
+    void basic_send(int target, int type, const void* data, int nbyte);
+    void basic_recv(int type, void* data, int nbyte);
+    int basic_probe(int type);
+    void initialize(int nprocs);
+    void initialize();
+
+    // previously static variables
+    commbuf_t *commbuf[MAXPROCS];
+    int shmid;
+    int semid;
+    int change_semid;
+    void* sharedmem;
+    struct sembuf semdec;
+    struct sembuf seminc;
+
+    // previously static functions for semephore operations
+    msgbuf_t *NEXT_MESSAGE(msgbuf_t *m);
+    void get_change(int node);
+    void put_change(int node);
+    void wait_for_write(int node);
+    void release_write(int node);
+#ifdef DEBUG
+    void print_buffer(int node, int me);
+#endif
+  public:
+    /// Reads the number of processors from environmental variable NUMPROC.
+    ShmMessageGrp();
+    /** The ShmMessageGrp KeyVal constructor takes a single keyword that
+       specifies the number of processors.  Here is an example of a
+       ParsedKeyVal input that creates a ShmMessageGrp that runs on four
+       processors:
+
+       <pre>
+       message<ShmMessageGrp>: n = 4
+       </pre>
+    */
+    ShmMessageGrp(const Ref<KeyVal>&);
+    /// Initialize ShmMessageGrp to use nprocs processors.
+    ShmMessageGrp(int nprocs);
+    ~ShmMessageGrp();
+    void sync();
+
+    Ref<MessageGrp> clone(void);
+};
+
+}
+     
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/messtest.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/messtest.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/messtest.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,329 @@
+//
+// messtest.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#include <util/misc/formio.h>
+#include <util/keyval/keyval.h>
+#include <util/class/class.h>
+#include <util/state/state.h>
+#include <util/misc/bug.h>
+#include <util/group/message.h>
+#include <util/group/mstate.h>
+#include <util/group/hcube.h>
+
+using namespace std;
+using namespace sc;
+
+// Force linkages:
+//#ifndef __PIC__
+#ifndef PUMAGON
+#   include <util/group/messshm.h>
+    static ForceLink<ShmMessageGrp> fl0;
+#endif
+# ifdef HAVE_MPI
+#   include <util/group/messmpi.h>
+    static ForceLink<MPIMessageGrp> fl2;
+# endif
+//#endif
+
+class A: virtual public SavableState {
+  private:
+    int ia;
+    int n;
+    int* array;
+    double d;
+  public:
+    A(int size);
+    A(const Ref<KeyVal>&);
+    A(StateIn&);
+    ~A();
+    void save_data_state(StateOut&);
+    inline int& a() { return ia; };
+    virtual void print (ostream&s = cout)
+    {
+      s << "A::a = " << a() << '\n';
+      s << "A::array = {";
+      for (int i=0; i<n; i++) s << array[i] << ' ';
+      s << "}\n";
+    }
+};
+
+A::A(int size):
+  ia(1),
+  n(size),
+  array(new int[size]),
+  d(-1.24)
+{
+  for (int i=0; i<size; i++) array[i] = size - i - 1;
+}
+A::A(const Ref<KeyVal>&keyval):
+  ia(keyval->intvalue("a")),
+  n(keyval->intvalue("n")),
+  d(-1.24)
+
+{
+  array = new int[n];
+  for (int i=0; i<n; i++) array[i] = i + 10000;
+}
+A::A(StateIn&s):
+  SavableState(s)
+{
+  cout << "getting d" << endl;
+  s.get(d);
+  cout << "getting ia" << endl;
+  s.get(ia);
+  cout << "getting array" << endl;
+  s.get(n);
+  s.get(array);
+  cout << "got everything" << endl;
+}
+A::~A()
+{
+  delete[] array;
+}
+void
+A::save_data_state(StateOut&s)
+{
+  cout << "putting d" << endl;
+  s.put(d);
+  cout << "putting ia" << endl;
+  s.put(ia);
+  cout << "putting array" << endl;
+  s.put(n);
+  s.put(array,n);
+  cout << "put everything" << endl;
+}
+
+static ClassDesc A_cd(
+  typeid(A),"A",1,"virtual public SavableState",
+  0, create<A>, create<A>);
+
+void test(const Ref<MessageGrp>&, int source, int target);
+void test_hcube(int nproc, int root, int fwd);
+
+int
+main(int argc, char**argv)
+{
+  Ref<MessageGrp> grp = MessageGrp::initial_messagegrp(argc, argv);
+
+  Ref<Debugger> debugger;
+
+  if (grp.null()) {
+      const char* input = SRCDIR "/messtest.in";
+      const char* keyword = "message";
+
+      if (argc >= 2) input = argv[1];
+      if (argc >= 3) keyword = argv[2];
+
+      Ref<KeyVal> keyval = new ParsedKeyVal(input);
+
+      grp << keyval->describedclassvalue(keyword);
+
+      debugger << keyval->describedclassvalue(":debug");
+
+      if (grp.null()) {
+          cerr << scprintf("Couldn't initialize MessageGrp\n");
+          abort();
+        }
+    }
+
+  if (debugger.nonnull()) {
+      debugger->set_exec(argv[0]);
+      debugger->set_prefix(grp->me());
+    }
+
+  Debugger::set_default_debugger(debugger);
+
+  grp->sync();
+  if (grp->n() > 1) {
+      BcastState bc(grp,1);
+      bc.bcast(debugger);
+      bc.flush();
+    }
+  grp->sync();
+  if (debugger.nonnull()) {
+      debugger->set_exec(argv[0]);
+      debugger->set_prefix(grp->me());
+      debugger->traceback();
+    }
+  grp->sync();
+
+  if (0 && grp->me() == 0) {
+      test_hcube(3, 0, 1);
+      test_hcube(39, 0, 1);
+      test_hcube(16, 0, 1);
+      test_hcube(17, 4, 1);
+      test_hcube(17, 4, 0);
+      test_hcube(1, 0, 0);
+    }
+
+  grp->sync();
+
+  if (grp->n() >= 3) {
+      test(grp, 2, 1);
+    }
+  else {
+      test(grp, 0, 0);
+    }
+
+  int testsum = 1;
+  grp->sum(&testsum,1);
+  if (testsum != grp->n()) {
+      cerr << scprintf("WARNING: sum wrong\n");
+    }
+
+  double testdsum = 1.0;
+  grp->sum(&testdsum,1);
+  cout << scprintf("on %d testdsum = %4.1f\n", grp->me(), testdsum);
+
+  grp->sync();
+  grp = 0;
+  return 0;
+}
+
+void
+test_hcube(int nproc, int root, int fwd)
+{
+  int i, j;
+  Ref<GlobalMsgIter> *gmi = new Ref<GlobalMsgIter>[nproc];
+  for (i=0; i<nproc; i++) {
+      gmi[i] =  new HypercubeGMI(nproc, i, root);
+    }
+  int iter = 1;
+  for (j=0; j<nproc; j++) {
+      if (fwd) {
+          gmi[j]->forwards();
+        }
+      else {
+          gmi[j]->backwards();
+        }
+    }
+  while (!gmi[0]->done()) {
+      cout << scprintf("------ step %d of %d ------\n", iter, gmi[0]->n());
+      for (j=0; j<nproc; j++) {
+          if (gmi[j]->send()) {
+              if (0 <= gmi[j]->sendto() && gmi[j]->sendto() < nproc) {
+                  if (gmi[gmi[j]->sendto()]->recvfrom() == j) {
+                      cout << scprintf(" %d -> %d\n", j, gmi[j]->sendto());
+                    }
+                  else {
+                      cout << scprintf(" %d -> (%d)\n", j, gmi[j]->sendto());
+                    }
+                }
+              else {
+                  cout << scprintf(" %d -> %d?\n", j, gmi[j]->sendto());
+                }
+            }
+          else if (gmi[j]->recv()) {
+              if (0 <= gmi[j]->recvfrom() && gmi[j]->recvfrom() < nproc) {
+                  if (gmi[gmi[j]->recvfrom()]->sendto() == j) {
+                      // to be printed by sender
+                    }
+                  else {
+                      cout << scprintf(" (%d) -> %d\n", gmi[j]->recvfrom(), j);
+                    }
+                }
+              else {
+                  cout << scprintf(" %d? -> %d\n", gmi[j]->recvfrom(), j);
+                }
+            }
+        }
+      for (j=0; j<nproc; j++) gmi[j]->next();
+      iter++;
+    }
+  cout.flush();
+}
+
+void
+test(const Ref<MessageGrp>& grp, int source, int target)
+{
+  Ref<A> a,b;
+  const int nca = 1000000;
+  char ca[nca];
+  
+  if (grp->me() == source) {
+      StateSend so(grp);
+      //so.set_buffer_size(5);
+      so.target(target);
+      a = new A(10);
+      SavableState::save_state(a,so);
+      so.flush();
+      grp->send(target, ca, nca);
+      if (source != target) grp->recv(target, ca, nca);
+    }
+
+  if (grp->me() == target) {
+      StateRecv si(grp);
+      //si.set_buffer_size(5);
+      si.source(source);
+      b << SavableState::restore_state(si);
+      if (source != target) grp->send(source, ca, nca);
+      grp->recv(source, ca, nca);
+    }
+
+  if (grp->me() == target) {
+      cout << "target:" << endl;
+      b->print();
+    }
+
+  grp->sync();
+
+  if (grp->me() == source) {
+      cout << "source:" << endl;
+      a->print();
+    }
+
+  ///////////////////////////////////////////////////
+  // Test broadcast
+
+  if (source != target) {
+      grp->sync();
+
+      b = 0;
+  
+      if (grp->me() == source) {
+          BcastStateSend so(grp);
+          SavableState::save_state(a,so);
+        }
+      else {
+          BcastStateRecv si(grp,source);
+          b << SavableState::restore_state(si);
+        }
+
+      if (grp->me() == target) {
+          cout << "bcast target:" << endl;
+          b->print();
+        }
+    }
+
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/messtest.in
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/messtest.in	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/messtest.in	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,17 @@
+% Emacs should use -*- KeyVal -*- mode.
+
+procmessage<ProcMessageGrp>: ()
+
+shmmessage<ShmMessageGrp>: (
+  n = 4
+  )
+
+mpimessage<MPIMessageGrp>: (
+  )
+
+message = $:shmmessage
+
+debug<Debugger>: (
+   cmd = "echo $(PREFIX) $(EXEC) $(PID)"
+   wait_for_debugger = no
+  )
Index: ThirdParty/mpqc_open/src/lib/util/group/mstate.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/mstate.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/mstate.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,697 @@
+//
+// mstate.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#include <util/misc/bug.h>
+#include <util/misc/formio.h>
+#include <util/group/mstate.h>
+
+#include <util/state/translate.h>
+
+using namespace std;
+using namespace sc;
+
+#define DEBUG 0
+
+// This sets up a communication buffer.  It is made up of a of
+// an integer that gives the number of bytes used in the buffer
+// by the data region of size bufsize.
+static
+void
+obtain_buffer(int*& nbuf_buffer, char*& send_buffer, int& nheader,
+              char*& buffer, int& bufsize, int size)
+{
+  if (size == bufsize) return;
+  if (send_buffer) delete[] (int*) send_buffer;
+
+  bufsize = size;
+
+  int min_bytes_to_allocate = bufsize + sizeof(int);
+  int ints_to_allocate = min_bytes_to_allocate/sizeof(int);
+  if (min_bytes_to_allocate%sizeof(int)) ints_to_allocate++;
+
+  nheader = sizeof(int);
+  int * isend_buffer = new int[ints_to_allocate];
+  send_buffer = (char*) isend_buffer;
+  buffer = (char*) & isend_buffer[1];
+  nbuf_buffer = isend_buffer;
+}
+
+static
+void
+release_buffer(char* send_buffer)
+{
+  if (send_buffer) delete[] (int*)send_buffer;
+}
+
+///////////////////////////////////////////////////////////////////////////
+// MsgStateSend member functions
+
+MsgStateSend::MsgStateSend(const Ref<MessageGrp>&grp_):
+  grp(grp_)
+{
+  nbuf = 0;
+  bufsize = 0;
+  send_buffer = 0;
+  node_to_node_ = 1;
+  obtain_buffer(nbuf_buffer,send_buffer,nheader,buffer,bufsize,8192);
+}
+
+MsgStateSend::~MsgStateSend()
+{
+  release_buffer(send_buffer);
+}
+
+void
+MsgStateSend::set_buffer_size(int size)
+{
+  flush();
+  obtain_buffer(nbuf_buffer,send_buffer,nheader,buffer,bufsize,size);
+}
+
+int
+MsgStateSend::put_array_void(const void* vd, int n)
+{
+  const char* d = (const char*) vd;
+  int remaining = n;
+
+  while (remaining) {
+      if (nbuf == bufsize) flush();
+      int ncurrent;
+      if (bufsize - nbuf < remaining) {
+          ncurrent = bufsize - nbuf;
+        }
+      else {
+          ncurrent = remaining;
+        }
+      memcpy(&buffer[nbuf],d,ncurrent);
+      remaining -= ncurrent;
+      nbuf += ncurrent;
+      d = &d[ncurrent];
+    }
+  return n;
+}
+
+int
+MsgStateSend::put(const ClassDesc*cd)
+{
+  int index = grp->classdesc_to_index(cd);
+  return StateOut::put(index);
+}
+
+int
+MsgStateSend::put(char d)
+{
+  return StateOut::put(d);
+}
+
+int
+MsgStateSend::put(unsigned int d)
+{
+  return StateOut::put(d);
+}
+
+int
+MsgStateSend::put(int d)
+{
+  return StateOut::put(d);
+}
+
+int
+MsgStateSend::put(float d)
+{
+  return StateOut::put(d);
+}
+
+
+int
+MsgStateSend::put(double d)
+{
+  return StateOut::put(d);
+}
+
+int
+MsgStateSend::put(const char* d, int n)
+{
+  return StateOut::put(d, n);
+}
+
+int
+MsgStateSend::put(const unsigned int* d, int n)
+{
+  return StateOut::put(d, n);
+}
+
+int
+MsgStateSend::put(const int* d, int n)
+{
+  return StateOut::put(d, n);
+}
+
+int
+MsgStateSend::put(const float* d, int n)
+{
+  return StateOut::put(d, n);
+}
+
+int
+MsgStateSend::put(const double* d, int n)
+{
+  return StateOut::put(d, n);
+}
+
+///////////////////////////////////////////////////////////////////////////
+// MsgStateBufRecv member functions
+
+static ClassDesc MsgStateBufRecv_cd(
+  typeid(MsgStateBufRecv),"MsgStateBufRecv",1,"public StateIn",
+  0, 0, 0);
+
+MsgStateBufRecv::MsgStateBufRecv()
+{
+  grp = MessageGrp::get_default_messagegrp();
+  nbuf = 0;
+  ibuf = 0;
+  send_buffer = 0;
+  bufsize = 0;
+  obtain_buffer(nbuf_buffer,send_buffer,nheader,buffer,bufsize,8192);
+}
+
+MsgStateBufRecv::MsgStateBufRecv(const Ref<MessageGrp>&grp_):
+  grp(grp_)
+{
+  nbuf = 0;
+  ibuf = 0;
+  send_buffer = 0;
+  bufsize = 0;
+  obtain_buffer(nbuf_buffer,send_buffer,nheader,buffer,bufsize,8192);
+}
+
+MsgStateBufRecv::~MsgStateBufRecv()
+{
+  if (ibuf && (nbuf != ibuf)) {
+      ExEnv::errn() << scprintf("MsgStateBufRecv::~MsgStateBufRecv(): buffer still has"
+              " %d bytes of data on %d\n", nbuf - ibuf, grp->me());
+    }
+  release_buffer(send_buffer);
+}
+
+void
+MsgStateBufRecv::set_buffer_size(int size)
+{
+  if (ibuf && (nbuf != ibuf)) {
+      ExEnv::errn() << "MsgStateBufRecv::set_buffer_size(): old buffer has data"
+           << endl;
+    }
+  obtain_buffer(nbuf_buffer, send_buffer, nheader, buffer, bufsize, size);
+}
+
+int
+MsgStateBufRecv::get_array_void(void* vd, int n)
+{
+  char* d = (char*) vd;
+
+  int remaining = n;
+
+  while (remaining) {
+      if (ibuf == nbuf) next_buffer();
+      int ncurrent;
+      if (nbuf - ibuf < remaining) {
+          ncurrent = nbuf - ibuf;
+        }
+      else {
+          ncurrent = remaining;
+        }
+      memcpy(d,&buffer[ibuf],ncurrent);
+      remaining -= ncurrent;
+      ibuf += ncurrent;
+      d = &d[ncurrent];
+    }
+
+  return n;
+}
+
+///////////////////////////////////////////////////////////////////////////
+// MsgStateRecv member functions
+
+MsgStateRecv::MsgStateRecv(const Ref<MessageGrp>&grp_):
+  MsgStateBufRecv(grp_)
+{
+  node_to_node_ = 1;
+}
+
+MsgStateRecv::~MsgStateRecv()
+{
+}
+
+int
+MsgStateRecv::version(const ClassDesc* cd)
+{
+  if (!cd) return -1;
+  return cd->version();
+}
+
+int
+MsgStateRecv::get(const ClassDesc**cd)
+{
+  int index;
+  int r = StateIn::get(index);
+  *cd = grp->index_to_classdesc(index);
+  if (!*cd) {
+      ExEnv::errn() << "MsgStateRecvt::get(const ClassDesc**cd): "
+           << "class not available on this processor:"
+           << endl;
+      ExEnv::errn() << " index = " << index << endl;
+      abort();
+    }
+  return r;
+}
+
+int
+MsgStateRecv::get(char& d, const char *key)
+{
+  return StateIn::get(d,key);
+}
+
+int
+MsgStateRecv::get(int& d, const char *key)
+{
+  return StateIn::get(d,key);
+}
+
+int
+MsgStateRecv::get(unsigned int& d, const char *key)
+{
+  return StateIn::get(d,key);
+}
+
+int
+MsgStateRecv::get(float& d, const char *key)
+{
+  return StateIn::get(d,key);
+}
+
+int
+MsgStateRecv::get(double& d, const char *key)
+{
+  return StateIn::get(d,key);
+}
+
+int
+MsgStateRecv::get(char*& d)
+{
+  return StateIn::get(d);
+}
+
+int
+MsgStateRecv::get(unsigned int*& d)
+{
+  return StateIn::get(d);
+}
+
+int
+MsgStateRecv::get(int*& d)
+{
+  return StateIn::get(d);
+}
+
+int
+MsgStateRecv::get(float*& d)
+{
+  return StateIn::get(d);
+}
+
+int
+MsgStateRecv::get(double*& d)
+{
+  return StateIn::get(d);
+}
+
+///////////////////////////////////////////////////////////////////////////
+// StateSend member functions
+
+StateSend::StateSend(const Ref<MessageGrp>&grp_):
+  MsgStateSend(grp_),
+  target_(0)
+{
+}
+
+StateSend::~StateSend()
+{
+  flush();
+}
+
+void
+StateSend::flush()
+{
+  if (nbuf == 0) return;
+  *nbuf_buffer = nbuf;
+  translate_->translator()->to_external(nbuf_buffer,1);
+  grp->raw_send(target_, send_buffer, nbuf + nheader);
+  nbuf = 0;
+}
+
+void
+StateSend::target(int t)
+{
+  target_ = t;
+  ps_.clear();
+}
+
+///////////////////////////////////////////////////////////////////////////
+// StateRecv member functions
+
+StateRecv::StateRecv(const Ref<MessageGrp>&grp_):
+  MsgStateRecv(grp_),
+  source_(0)
+{
+}
+
+void
+StateRecv::next_buffer()
+{
+  grp->raw_recv(source_, send_buffer, bufsize+nheader);
+  translate_->translator()->to_native(nbuf_buffer,1);
+  nbuf = *nbuf_buffer;
+  ibuf = 0;
+}
+
+void
+StateRecv::source(int s)
+{
+  source_ = s;
+  ps_.clear();
+}
+
+///////////////////////////////////////////////////////////////////////////
+// BcastStateSend member functions
+
+BcastStateSend::BcastStateSend(const Ref<MessageGrp>&grp_):
+  MsgStateSend(grp_)
+{
+}
+
+BcastStateSend::~BcastStateSend()
+{
+  flush();
+}
+
+void
+BcastStateSend::flush()
+{
+  if (nbuf == 0) return;
+  *nbuf_buffer = nbuf;
+  translate_->translator()->to_external(nbuf_buffer,1);
+  grp->raw_bcast(send_buffer, nbuf + nheader, grp->me());
+  nbuf = 0;
+}
+
+///////////////////////////////////////////////////////////////////////////
+// BcastStateRecv member functions
+
+BcastStateRecv::BcastStateRecv(const Ref<MessageGrp>&grp_, int s):
+  MsgStateRecv(grp_)
+{
+  source(s);
+}
+
+void
+BcastStateRecv::source(int s)
+{
+  if (s == grp->me()) {
+      ExEnv::errn() << scprintf("BcastStateRecv::source(%d): cannot receive my own"
+              " broadcast\n", s);
+      abort();
+    }
+  source_ = s;
+  ps_.clear();
+}
+
+void
+BcastStateRecv::next_buffer()
+{
+  grp->raw_bcast(send_buffer, bufsize+nheader, source_);
+  translate_->translator()->to_native(nbuf_buffer,1);
+  nbuf = *nbuf_buffer;
+  ibuf = 0;
+}
+
+///////////////////////////////////////////////////////////////////////////
+// BcastState member functions
+
+BcastState::BcastState(const Ref<MessageGrp> &grp, int source)
+{
+  if (grp->n() == 1) {
+      recv_ = 0;
+      send_ = 0;
+    }
+  else if (grp->me() == source) {
+      recv_ = 0;
+      send_ = new BcastStateSend(grp);
+    }
+  else {
+      recv_ = new BcastStateRecv(grp,source);
+      send_ = 0;
+    }
+}
+
+BcastState::~BcastState()
+{
+  delete recv_;
+  delete send_;
+}
+
+void
+BcastState::bcast(int &a)
+{
+  if (recv_) recv_->get(a);
+  else if (send_) send_->put(a);
+}
+
+void
+BcastState::bcast(double &a)
+{
+  if (recv_) recv_->get(a);
+  else if (send_) send_->put(a);
+}
+
+void
+BcastState::bcast(int *&a, int n)
+{
+  if (recv_) recv_->get(a);
+  else if (send_) send_->put(a,n);
+}
+
+void
+BcastState::bcast(double *&a, int n)
+{
+  if (recv_) recv_->get(a);
+  else if (send_) send_->put(a,n);
+}
+
+void
+BcastState::flush()
+{
+  if (send_) send_->flush();
+}
+
+void
+BcastState::set_buffer_size(int n)
+{
+  if (send_) send_->set_buffer_size(n);
+  if (recv_) recv_->set_buffer_size(n);
+}
+
+void
+BcastState::forget_references()
+{
+  if (send_) send_->forget_references();
+}
+
+///////////////////////////////////////////////////////////////////////////
+// BcastStateRecv member functions
+
+static ClassDesc BcastStateInBin_cd(
+  typeid(BcastStateInBin),"BcastStateInBin",1,"public MsgStateBufRecv",
+  0, create<BcastStateInBin>, 0);
+
+BcastStateInBin::BcastStateInBin(const Ref<MessageGrp>&grp_,
+                                 const char *filename):
+  MsgStateBufRecv(grp_)
+{
+  opened_ = 0;
+  open(filename);
+}
+
+BcastStateInBin::BcastStateInBin(const Ref<KeyVal> &keyval)
+{
+  char *path = keyval->pcharvalue("file");
+  if (!path) {
+      ExEnv::errn() << "StateInBin(const Ref<KeyVal>&): no path given" << endl;
+    }
+  opened_ = 0;
+  open(path);
+  delete[] path;
+}
+
+BcastStateInBin::~BcastStateInBin()
+{
+  close();
+}
+
+void
+BcastStateInBin::next_buffer()
+{
+  if (grp->me() == 0) {
+      // fill the buffer
+#if HAVE_SGETN
+      *nbuf_buffer = buf_->sgetn(buffer,bufsize);
+#else
+      *nbuf_buffer = buf_->xsgetn(buffer,bufsize);
+#endif
+      if (*nbuf_buffer == 0) {
+          ExEnv::errn() << "BcastStateInBin: read failed" << endl;
+          abort();
+        }
+      translate_->translator()->to_external(nbuf_buffer,1);
+    }
+  grp->raw_bcast(send_buffer, bufsize+nheader);
+  translate_->translator()->to_native(nbuf_buffer,1);
+  nbuf = *nbuf_buffer;
+  ibuf = 0;
+}
+
+void
+BcastStateInBin::close()
+{
+  if(opened_) delete buf_;
+  opened_=0; buf_=0;
+  nbuf = 0;
+  ibuf = 0;
+
+  classidmap_.clear();
+  nextclassid_ = 0;
+  classdatamap_.clear();
+  ps_.clear();
+}
+
+int
+BcastStateInBin::open(const char *path)
+{
+  file_position_ = 0;
+
+  if (grp->me() == 0) { 
+      if (opened_) close();
+
+      filebuf *fbuf = new filebuf();
+      fbuf->open(path, ios::in);
+      if (!fbuf->is_open()) {
+          ExEnv::errn() << "ERROR: BcastStateInBin: problems opening " << path << endl;
+          abort();
+        }
+      buf_ = fbuf;
+      opened_ = 1;
+    }
+
+  nbuf = 0;
+  ibuf = 0;
+
+  get_header();
+  find_and_get_directory();
+
+  return 0;
+}
+
+int
+BcastStateInBin::tell()
+{
+  return file_position_;
+}
+
+void
+BcastStateInBin::seek(int loc)
+{
+  file_position_ = loc;
+#if defined(HAVE_PUBSEEKOFF)
+  if (grp->me() == 0) {
+      buf_->pubseekoff(loc,ios::beg,ios::in);
+#  if  DEBUG
+      ExEnv::outn() << "pubseekoff to " << loc << endl;
+#  endif
+    }
+#elif defined(HAVE_SEEKOFF)
+  if (grp->me() == 0) {
+      buf_->seekoff(loc,ios::beg,ios::in);
+#  if  DEBUG
+      ExEnv::outn() << "seekoff to " << loc << endl;
+#  endif
+    }
+#endif
+  nbuf = 0;
+  ibuf = 0;
+}
+
+int
+BcastStateInBin::seekable()
+{
+#if defined(HAVE_PUBSEEKOFF) || defined(HAVE_SEEKOFF)
+  return 1;
+#else
+  return 0;
+#endif
+}
+
+int
+BcastStateInBin::use_directory()
+{
+  return seekable();
+}
+
+int
+BcastStateInBin::get_array_void(void* vd, int n)
+{
+  MsgStateBufRecv::get_array_void(vd, n);
+  file_position_ += n;
+#if DEBUG
+  ExEnv::outn() << "Read " << n << " bytes:";
+  for (int i=0; i<n; i++) {
+      ExEnv::outn() << " " << (int) ((unsigned char*)vd)[i];
+    }
+  ExEnv::outn() << endl;
+#endif
+  return n;
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/mstate.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/mstate.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/mstate.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,310 @@
+//
+// mstate.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_mstate_h
+#define _util_group_mstate_h
+
+#include <util/state/state.h>
+#include <util/state/statein.h>
+#include <util/state/stateout.h>
+#include <util/group/message.h>
+
+namespace sc {
+
+/** The MsgStateSend is an abstract base class that sends objects
+    to nodes in a MessageGrp.
+*/
+class MsgStateSend: public StateOut {
+  private:
+    // do not allow copy constructor or assignment
+    MsgStateSend(const MsgStateSend&);
+    void operator=(const MsgStateSend&);
+  protected:
+    Ref<MessageGrp> grp;
+    int nbuf; // the number of bytes used in the buffer
+    int bufsize; // the allocated size of the data buffer
+    char* buffer; // the data buffer
+    char* send_buffer; // the buffer used to send data (includes nbuf)
+    int nheader; // nbuf + nheader = the number of bytes in send_buffer to send
+    int* nbuf_buffer; // the pointer to the nbuf stored in the buffer
+
+    int put_array_void(const void*, int);
+  public:
+    MsgStateSend(const Ref<MessageGrp>&);
+    virtual ~MsgStateSend();
+
+    /// Specializations must implement flush().
+    virtual void flush() = 0;
+
+    /** The buffer size of statein and stateout objects that communicate
+        with each other must match. */
+    void set_buffer_size(int);
+
+    /** I only need to override put(const ClassDesc*) but C++ will
+        hide all of the other put's so I must override everything. */
+    int put(const ClassDesc*);
+    int put(char r);
+    int put(unsigned int r);
+    int put(int r);
+    int put(float r);
+    int put(double r);
+    int put(const char*,int);
+    int put(const int*,int);
+    int put(const unsigned int*,int);
+    int put(const float*,int);
+    int put(const double*,int);
+};
+
+/** The MsgStateBufRecv is an abstract base class that
+    buffers objects sent through a MessageGrp.
+*/
+class MsgStateBufRecv: public StateIn {
+  private:
+    // do not allow copy constructor or assignment
+    MsgStateBufRecv(const MsgStateBufRecv&);
+    void operator=(const MsgStateBufRecv&);
+  protected:
+    Ref<MessageGrp> grp;
+    int nbuf; // the number of bytes used in the buffer
+    int ibuf; // the current pointer withing the buffer
+    int bufsize; // the allocated size of the buffer
+    char* buffer; // the data buffer
+    char* send_buffer; // the buffer used to send data (includes nbuf)
+    int nheader; // nbuf + nheader = the number of bytes in send_buffer to send
+    int* nbuf_buffer; // the pointer to the nbuf stored in the buffer
+
+    int get_array_void(void*,int);
+
+    /// Specializations must implement next_buffer().
+    virtual void next_buffer() = 0;
+  public:
+    /// MsgStateBufRecv can be initialized with a MessageGrp.
+    MsgStateBufRecv(const Ref<MessageGrp>&);
+    /// Use the default MessageGrp.
+    MsgStateBufRecv();
+
+    virtual ~MsgStateBufRecv();
+
+    /** The buffer size of statein and stateout objects that communicate
+        with each other must match. */
+    void set_buffer_size(int);
+};
+
+/** The MsgStateRecv is an abstract base class that receives
+    objects from nodes in a MessageGrp. */
+class MsgStateRecv: public MsgStateBufRecv {
+  private:
+    // do not allow copy constructor or assignment
+    MsgStateRecv(const MsgStateRecv&);
+    void operator=(const MsgStateRecv&);
+  public:
+    /// MsgStateRecv must be initialized with a MessageGrp.
+    MsgStateRecv(const Ref<MessageGrp>&);
+
+    virtual ~MsgStateRecv();
+
+    /** Returns the version of the ClassDesc.  This assumes that
+        the version of the remote class is the same as that of
+        the local class. */
+    int version(const ClassDesc*);
+
+    /** I only need to override get(ClassDesc**) but C++ will hide
+        all of the other get's so I must override everything. */
+    int get(const ClassDesc**);
+    int get(char&r, const char *key = 0);
+    int get(unsigned int&r, const char *key = 0);
+    int get(int&r, const char *key = 0);
+    int get(float&r, const char *key = 0);
+    int get(double&r, const char *key = 0);
+    int get(char*&);
+    int get(unsigned int*&);
+    int get(int*&);
+    int get(float*&);
+    int get(double*&);
+};
+
+/** StateSend is a concrete specialization of
+    MsgStateSend that does the send part of point to
+    point communication in a MessageGrp. */
+class StateSend: public MsgStateSend {
+  private:
+    // do not allow copy constructor or assignment
+    StateSend(const StateSend&);
+    void operator=(const StateSend&);
+  private:
+    int target_;
+  public:
+    /// Create a StateSend given a MessageGrp.
+    StateSend(const Ref<MessageGrp>&);
+
+    ~StateSend();
+    /// Specify the target node.
+    void target(int);
+    /// Flush the buffer.
+    void flush();
+};
+
+/** StateRecv is a concrete specialization of
+    MsgStateRecv that does the receive part of point to
+    point communication in a MessageGrp. */
+class StateRecv: public MsgStateRecv {
+  private:
+    // do not allow copy constructor or assignment
+    StateRecv(const StateRecv&);
+    void operator=(const StateRecv&);
+  private:
+    int source_;
+  protected:
+    void next_buffer();
+  public:
+    /// Create a StateRecv given a MessageGrp.
+    StateRecv(const Ref<MessageGrp>&);
+    /// Specify the source node.
+    void source(int);
+};
+
+/** BcastStateSend does the send part of a broadcast of an object
+    to all nodes.  Only one node uses a BcastStateSend and the rest
+    must use a BcastStateRecv. */
+class BcastStateSend: public MsgStateSend {
+  private:
+    // do not allow copy constructor or assignment
+    BcastStateSend(const BcastStateSend&);
+    void operator=(const BcastStateSend&);
+  public:
+    /// Create the BcastStateSend.
+    BcastStateSend(const Ref<MessageGrp>&);
+
+    ~BcastStateSend();
+    /// Flush the data remaining in the buffer.
+    void flush();
+};
+
+/** BcastStateRecv does the receive part of a broadcast of an
+    object to all nodes.  Only one node uses a BcastStateSend and
+    the rest must use a BcastStateRecv. */
+class BcastStateRecv: public MsgStateRecv {
+  private:
+    // do not allow copy constructor or assignment
+    BcastStateRecv(const BcastStateRecv&);
+    void operator=(const BcastStateRecv&);
+  protected:
+    int source_;
+    void next_buffer();
+  public:
+    /// Create the BcastStateRecv.
+    BcastStateRecv(const Ref<MessageGrp>&, int source = 0);
+    /// Set the source node.
+    void source(int s);
+};
+
+/** This creates and forwards/retrieves data from either a BcastStateRecv
+    or a BcastStateSend depending on the value of the argument to
+    constructor. */
+class BcastState {
+  private:
+    BcastStateRecv *recv_;
+    BcastStateSend *send_;
+  public:
+    /// Create a BcastState object.  The default source is node 0.
+    BcastState(const Ref<MessageGrp> &, int source = 0);
+
+    ~BcastState();
+
+    /** Broadcast data to all nodes.  After these are called
+        for a group of data the flush member must be called
+        to force the source node to actually write the data. */
+    void bcast(int &);
+    void bcast(double &);
+    void bcast(int *&, int);
+    void bcast(double *&, int);
+    template <class T> void bcast(Ref<T>&a)
+        {
+          if (recv_) {
+              a << SavableState::restore_state(*recv_);
+            }
+          else if (send_) {
+              SavableState::save_state(a.pointer(),*send_);
+            }
+        }
+
+    /** Force data to be written.  Data is not otherwise written
+        until the buffer is full. */
+    void flush();
+
+    /** Call the StateOut or StateIn
+        forget_references member. */
+    void forget_references();
+
+    /// Controls the amount of data that is buffered before it is sent.
+    void set_buffer_size(int);
+};
+
+/** BcastStateBin reads a file in written by
+    StateInBin on node 0 and broadcasts it to all nodes
+    so state can be simultaneously restored on all nodes. */
+class BcastStateInBin: public MsgStateBufRecv {
+  private:
+    // do not allow copy constructor or assignment
+    BcastStateInBin(const BcastStateRecv&);
+    void operator=(const BcastStateRecv&);
+  protected:
+    int opened_;
+    int file_position_;
+    std::streambuf *buf_;
+
+    void next_buffer();
+    int get_array_void(void*, int);
+  public:
+    /// Create the BcastStateRecv using the default MessageGrp.
+    BcastStateInBin(const Ref<KeyVal> &);
+    /// Create the BcastStateRecv.
+    BcastStateInBin(const Ref<MessageGrp>&, const char *filename);
+
+    ~BcastStateInBin();
+
+    virtual int open(const char *name);
+    virtual void close();
+
+    void seek(int loc);
+    int seekable();
+    int tell();
+    int use_directory();
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/pool.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/pool.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/pool.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,318 @@
+//
+// pool.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#include <util/misc/formio.h>
+#include <util/group/pool.h>
+
+using namespace std;
+using namespace sc;
+
+void
+PoolData::check(void* lower_bound, void* upper_bound)
+{
+  if ((void*)this < lower_bound || (void*)this >= upper_bound) {
+      ExEnv::errn() << scprintf("PoolData::check: this out of bounds\n");
+      abort();
+    }
+  if (next_) {
+      if ((void*)next_ < lower_bound || (void*)next_ >= upper_bound) {
+          ExEnv::errn() << scprintf("PoolData::check: next_ out of bounds\n");
+          abort();
+        }
+      if (next_->prev_ != this) {
+          ExEnv::errn() << scprintf("PoolData::check: next pd doesn't point back\n");
+          abort();
+        }
+      if ((char*)next_ != (char*)this + size_ + PoolData_aligned_size) {
+          ExEnv::errn() << scprintf("PoolData::check: next_ not consistent with size\n");
+          abort();
+        }
+      if (free_ && next_->free_) {
+          ExEnv::errn() << scprintf("PoolData::check: free and next is free\n");
+          abort();
+        }
+    }
+  if (prev_) {
+      if ((void*)prev_ < lower_bound || (void*)prev_ >= upper_bound) {
+          ExEnv::errn() << scprintf("PoolData::check: prev_ out of bounds\n");
+          abort();
+        }
+      if (prev_->next_ != this) {
+          ExEnv::errn() << scprintf("PoolData::check: prev pd doesn't point back\n");
+          abort();
+        }
+      if (free_ && prev_->free_) {
+          ExEnv::errn() << scprintf("PoolData::check: free and prev is free\n");
+          abort();
+        }
+    }
+  if (free_) {
+      PoolData* n = f.next_free_;
+      PoolData* p = f.prev_free_;
+      if (n) {
+          if ((void*)n < lower_bound || (void*)n >= upper_bound) {
+              ExEnv::errn() << scprintf("PoolData::check: next free out of bounds\n");
+              abort();
+            }
+          if (n->f.prev_free_ != this) {
+              ExEnv::errn() << scprintf(
+                      "PoolData::check: next free pd doesn't point back\n");
+              abort();
+            }
+        }
+      if (p) {
+          if ((void*)p < lower_bound || (void*)p >= upper_bound) {
+              ExEnv::errn() << scprintf("PoolData::check: prev free out of bounds\n");
+              abort();
+            }
+          if (p->f.next_free_ != this) {
+              ExEnv::errn() << scprintf(
+                      "PoolData::check: prev free pd doesn't point back\n");
+              abort();
+            }
+        }
+    }
+}
+
+Pool::Pool(size_t size):
+  size_(size)
+{
+
+  // Initialize the first and last members of the data list.
+  firstdatum_ = (PoolData*)align_pool_data((void*)((char*)this+sizeof(Pool)));
+
+  if ((char*)this + size <= (char*) firstdatum_) {
+      ExEnv::errn() << scprintf("Pool::Pool: not given enough space\n");
+      abort();
+    }
+  
+  size_t firstdatum_size = align_pool_data_downward((size_t)
+                                                    (((char*)this+size)
+                                                    - (char*)firstdatum_));
+  new(firstdatum_) PoolData(firstdatum_size);
+
+  firstdatum_->prev_next(0,0);
+
+  // Initialize the free lists.
+  int i;
+  for (i=0; i<freelist_size; i++) freelist_[i] = 0;
+  freelist_add(firstdatum_);
+}
+
+void
+Pool::freelist_add(PoolData*d)
+{
+  int slot = freelist_find_slot(d->size_);
+  d->free_ = 1;
+  PoolData* tmp = freelist_[slot];
+  d->next_free(tmp);
+  d->prev_free(0);
+  freelist_[slot] = d;
+  if (tmp) tmp->prev_free(d);
+#ifdef DEBUG_POOL
+  d->check();
+  if (d->next()) d->next()->check();
+  if (d->prev()) d->prev()->check();
+#endif
+}
+
+void
+Pool::freelist_del(PoolData*d)
+{
+  if (d->next_free()) d->next_free()->prev_free(d->prev_free());
+  if (d->prev_free()) d->prev_free()->next_free(d->next_free());
+  else {
+      int slot = freelist_find_slot(d->size_);
+      freelist_[slot] = d->next_free();
+    }
+  d->free_ = 0;
+#ifdef DEBUG_POOL
+  d->check();
+  if (d->next()) d->next()->check();
+  if (d->prev()) d->prev()->check();
+#endif
+}
+
+int
+Pool::freelist_find_slot(size_t size)
+{
+  int slot = 0;
+  size_t mask = ~ (size_t)0;
+  while(mask & size) {
+      slot++;
+      mask <<= 1;
+    }
+  return slot;
+}
+
+void*
+Pool::allocate(size_t size)
+{
+  int slot = freelist_find_slot(size);
+  for (int i=slot; i<freelist_size; i++) {
+      PoolData* j;
+      for (j=freelist_[i]; j; j = j->next_free()) {
+          if (j->size_ >= size) {
+              freelist_del(j);
+              // Maybe need to break this chunk into two pieces.
+              if (j->size_ > size + PoolData_aligned_size) {
+                  PoolData* freechunk = (PoolData*)((char*)j
+                                                    + PoolData_aligned_size
+                                                    + size);
+                  new(freechunk) PoolData(j->size_ - size);
+                  freechunk->prev_next(j,j->next());
+                  if (freechunk->next()) freechunk->next()->prev(freechunk);
+                  j->size_ = size;
+                  j->next(freechunk);
+                  freelist_add(freechunk);
+                }
+#ifdef DEBUG_POOL
+              j->check();
+              if (j->next()) j->next()->check();
+              if (j->prev()) j->prev()->check();
+#endif
+              return j->data();
+            }
+        }
+    }
+  return 0;
+}
+
+void
+Pool::release(void* v)
+{
+  PoolData *d = voidptr_to_pd(v);
+  if (d->prev() && d->prev()->free_) {
+      freelist_del(d->prev());
+      d->prev()->size_ += d->size_ + PoolData_aligned_size;
+      d->prev()->next(d->next());
+      if (d->next()) d->next()->prev(d->prev());
+      d->set_magic(0);
+      d = d->prev();
+    }
+  if (d->next() && d->next()->free_) {
+      freelist_del(d->next());
+      d->next()->set_magic(0);
+      d->size_ += d->next()->size_ + PoolData_aligned_size;
+      if (d->next()->next()) d->next()->next()->prev(d);
+      d->next(d->next()->next());
+    }
+  freelist_add(d);
+}
+
+static void
+print_pooldata(ostream&o,PoolData*d,int free)
+{
+  PoolData *next,*prev;
+  if (free) {
+      next = d->next_free();
+      prev = d->prev_free();
+    }
+  else {
+      next = d->next();
+      prev = d->prev();
+    }
+
+  o << scprintf("    PoolData: size=%d", d->size_);
+  if (d->free_) o << scprintf(", free");
+  else o << scprintf(", allocated");
+  if (!prev) o << scprintf(", first");
+  if (!next) o << scprintf(", last");
+  o << endl;
+  if (next) print_pooldata(o,next,free);
+}
+
+void
+Pool::print(ostream&o)
+{
+  o << scprintf("Memory Pool:\n");
+  o << scprintf("  data chain:\n");
+  print_pooldata(o,firstdatum_,0);
+  for (int i=0; i<freelist_size; i++) {
+      if (freelist_[i]) {
+          o << scprintf("  freelist[%d]:\n",i);
+          print_pooldata(o,freelist_[i],1);
+        }
+    }
+}
+
+void
+Pool::check()
+{
+  // The bit lost at the beginning to Pool and alignment.
+  size_t start = (size_t)
+                 ((char*)align_pool_data((void*)((char*)this + sizeof(Pool)))
+                  - (char*)this);
+
+  // The bit lost at the end to alignment.
+  size_t end = (size_t)
+               ((char*)this + size_
+                - (char*) align_pool_data_downward((size_t)((void*)this)
+                                                   +size_));
+
+  size_t size = start + end;
+
+  PoolData *j;
+  for (j=firstdatum_; j; j = j->next()) {
+      j->check(this,(void*)((char*)this+size_));
+      size += PoolData_aligned_size + j->size_;
+    }
+  
+  if (size != size_) {
+      ExEnv::errn() << scprintf("Pool::check(): inconsistent sizes\n");
+      ExEnv::errn() << scprintf("  computed: %d\n",size);
+      ExEnv::errn() << scprintf("  actual:   %d\n",size_);
+      abort();
+    }
+
+  // make sure that all data is accounted for
+  for (j=firstdatum_; j; j = j->next()) {
+      j->flags_ = 1;
+    }
+  for (int i=0; i<freelist_size; i++) {
+      for (j=freelist_[i]; j; j = j->next_free()) {
+          j->flags_ = 0;
+        }
+    }
+  for (j=firstdatum_; j; j = j->next()) {
+      if (j->free_ && j->flags_) {
+          ExEnv::errn() << scprintf("Pool::check: free data not in freelist\n");
+          abort();
+        }
+    }
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/pool.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/pool.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/pool.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,336 @@
+//
+// pool.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_pool_h
+#define _util_group_pool_h
+
+#include <stdlib.h>
+#include <new>
+#include <iostream>
+
+#include <util/misc/exenv.h>
+
+#undef DEBUG_POOL
+
+namespace sc {
+
+const int pool_data_alignment_bit = 3;
+//const int pool_data_alignment_bit = 14;
+const size_t pool_data_alignment = 1<<pool_data_alignment_bit;
+inline size_t
+align_pool_data(size_t size)
+{
+  return (size + pool_data_alignment - 1)
+      & (~ (pool_data_alignment - 1));
+}
+inline void*
+align_pool_data(void* ptr)
+{
+  return (void*)( (unsigned long) ((char*)ptr + pool_data_alignment - 1)
+                 & (~ (pool_data_alignment - 1)));
+}
+inline size_t
+align_pool_data_downward(size_t size)
+{
+  return size & (~ (pool_data_alignment - 1));
+}
+inline void*
+align_pool_data_downward(void* ptr)
+{
+  return (void*) ( (unsigned long) ptr & (~ (pool_data_alignment - 1)));
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+
+class PoolData;
+struct FreeData {
+    PoolData* next_free_;
+    PoolData* prev_free_;
+};
+
+// ////////////////////////////////////////////////////////////////////////////
+
+struct UsedData {
+    unsigned int flags;
+    unsigned int held_:16;
+    int priority_:15;
+    unsigned int fixed_:1;
+};
+
+// ////////////////////////////////////////////////////////////////////////////
+
+class PoolData {
+  public:
+    enum {magic = 0x1f1d1e1c};
+    int magic_;
+    size_t size_;
+    unsigned int free_:1;
+    unsigned int flags_:15;
+  private:
+    PoolData* next_;
+    PoolData* prev_;
+  public:
+    union {
+        FreeData f;
+        UsedData u;
+    };
+
+    // Allocates a chunk of free memory, only initializing the size.
+    PoolData(size_t size);
+
+    PoolData* next();
+    PoolData* prev();
+
+    void next(PoolData*);
+    void prev(PoolData*);
+    void prev_next(PoolData*,PoolData*);
+
+    PoolData* next_free();
+    PoolData* prev_free();
+
+    void next_free(PoolData*);
+    void prev_free(PoolData*);
+    void prev_next_free(PoolData*,PoolData*);
+
+    void set_magic(int = magic);
+
+    // This new can only be called with aligned memory.
+    //void* operator new(size_t size, void* placement);
+    void* data();
+
+    void check(void*lower=(void*)0x0,void*upper=(void*)0x7fffffffL);
+};
+
+const int PoolData_aligned_size = (sizeof(PoolData) + pool_data_alignment - 1)
+    & (~ (pool_data_alignment - 1));
+inline void* PoolData::data()
+{
+  return (void*)(((char*)this) + PoolData_aligned_size);
+}
+
+inline PoolData*
+PoolData::next()
+{
+  return next_;
+}
+
+inline PoolData*
+PoolData::prev()
+{
+  return prev_;
+}
+
+inline void
+PoolData::next(PoolData*p)
+{
+  next_ = p;
+#ifdef DEBUG_POOL
+  if (next_ && prev_ && (next_ < prev_)) {
+      ExEnv::errn() << "PoolData::next(PoolData*): next < prev" << endl;
+      abort();
+    }
+#endif
+}
+
+inline void
+PoolData::prev(PoolData*p)
+{
+  prev_ = p;
+#ifdef DEBUG_POOL
+  if (next_ && prev_ && (next_ < prev_)) {
+      ExEnv::errn() << "PoolData::prev(PoolData*): next < prev" << endl;
+      abort();
+    }
+#endif
+}
+
+inline void
+PoolData::prev_next(PoolData*p,PoolData*n)
+{
+  prev_ = p;
+  next_ = n;
+#ifdef DEBUG_POOL
+  if (next_ && prev_ && (next_ < prev_)) {
+      ExEnv::errn() << "PoolData::prev_next: next < prev" << endl;
+      abort();
+    }
+#endif
+}
+
+// ////
+
+inline PoolData*
+PoolData::next_free()
+{
+#ifdef DEBUG_POOL
+  if (!free_) {
+      ExEnv::errn() << "PoolData::next_free(): datum is not free" << endl;
+      abort();
+    }
+#endif
+  return f.next_free_;
+}
+
+inline PoolData*
+PoolData::prev_free()
+{
+#ifdef DEBUG_POOL
+  if (!free_) {
+      ExEnv::errn() << "PoolData::prev_free(): datum is not free" << endl;
+      abort();
+    }
+#endif
+  return f.prev_free_;
+}
+
+inline void
+PoolData::next_free(PoolData*p)
+{
+#ifdef DEBUG_POOL
+  if (!free_) {
+      ExEnv::errn() << "PoolData::next_free(PoolData*): datum is not free" << endl;
+      abort();
+    }
+#endif
+  f.next_free_ = p;
+}
+
+inline void
+PoolData::prev_free(PoolData*p)
+{
+#ifdef DEBUG_POOL
+  if (!free_) {
+      ExEnv::errn() << "PoolData::prev_free(PoolData*): datum is not free" << endl;
+      abort();
+    }
+#endif
+  f.prev_free_ = p;
+}
+
+inline void
+PoolData::prev_next_free(PoolData*p,PoolData*n)
+{
+#ifdef DEBUG_POOL
+  if (!free_) {
+      ExEnv::errn() << "PoolData::prev_next_free: datum is not free" << endl;
+      abort();
+    }
+#endif
+  f.prev_free_ = p;
+  f.next_free_ = n;
+}
+
+inline
+PoolData::PoolData(size_t size):
+  magic_(magic),
+  size_(size-PoolData_aligned_size)
+{
+}
+
+inline void
+PoolData::set_magic(int magic_a)
+{
+  magic_ = magic_a;
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+
+class Pool {
+  protected:
+    enum { freelist_size = sizeof(size_t)*8 };
+    PoolData* freelist_[freelist_size];
+
+    size_t size_;
+
+    PoolData* firstdatum_;
+    PoolData* voidptr_to_pd(void*d);
+
+    int freelist_find_slot(size_t);
+    void freelist_add(PoolData*);
+    void freelist_del(PoolData*);
+  public:
+    Pool(size_t);
+    ~Pool();
+    
+//     void* operator new(size_t size, void* placement) { return placement; }
+
+//     Handle& allocate_handle(size_t size, int priority = 0);
+//     void release(Handle&);
+
+    size_t size() { return size_; }
+
+    void* allocate(size_t size);
+    void release(void*d);
+    double* allocate_double(size_t n);
+    void release(double*d);
+    int* allocate_int(size_t n);
+    void release(int*d);
+    void print(std::ostream&o=ExEnv::out0());
+    void check();
+};
+
+inline PoolData*
+Pool::voidptr_to_pd(void*d)
+{
+  return (PoolData*)((char*)d - PoolData_aligned_size);
+}
+
+inline double*
+Pool::allocate_double(size_t n)
+{
+  return (double*) allocate(n*sizeof(double));
+}
+
+inline void
+Pool::release(double*d)
+{
+  release((void*)d);
+}
+inline int*
+Pool::allocate_int(size_t n)
+{
+  return (int*) allocate(n*sizeof(int));
+}
+inline void
+Pool::release(int*d)
+{
+  release((void*)d);
+}
+
+}
+
+#endif
+
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/pooltest.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/pooltest.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/pooltest.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,275 @@
+//
+// pooltest.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#include <stdlib.h>
+#include <math.h>
+
+#include <util/group/pool.h>
+#include <util/misc/formio.h>
+
+using namespace std;
+using namespace sc;
+
+class Double {
+  private:
+    static Pool* pool_;
+    static Double* list;
+    Double* next;
+    Double* prev;
+    double* d;
+    int size;
+    static void zaplist(Double*);
+  public:
+    Double(size_t size);
+    ~Double();
+    void zap();
+    static void zapall();
+    void clear();
+    static void pool(Pool*);
+};
+
+Double* Double::list = 0;
+Pool* Double::pool_ = 0;
+
+void
+Double::clear()
+{
+  if (d) pool_->release(d);
+  d = 0;
+  size = 0;
+}
+
+void
+Double::zapall()
+{
+  zaplist(list);
+}
+
+void
+Double::zaplist(Double*l)
+{
+  for (Double* i=l; i; i = i->next) {
+      i->zap();
+    }
+}
+
+Double::Double(size_t s):
+  size(s)
+{
+  if (!pool_) {
+      cerr << scprintf("Double::Double: Pool not initialized\n");
+      abort();
+    }
+  d = pool_->allocate_double(size);
+  if (!d) {
+      //cout << scprintf("\nDouble::Double allocation of size %d failed\n",
+      //                 size);
+      cout << "F" << endl;
+      size = 0;
+    }
+  next = list;
+  prev = 0;
+  list = this;
+  if (next) next->prev = this;
+}
+
+Double::~Double()
+{
+  clear();
+  if (next) next->prev = prev;
+  if (prev) prev->next = next;
+  else list = next;
+}
+
+void
+Double::zap()
+{
+  int i;
+  int* x = (int*)d;
+  for (i=0; i<size*2; i++) {
+      if (x[i] == PoolData::magic) {
+          cerr << scprintf("Double::zap: tried to zap a magic number\n");
+          abort();
+        }
+    }
+  for (i=0; i<size; i++) d[i] = 0.0;
+}
+
+void
+Double::pool(Pool*p)
+{
+  if (pool_ && list) {
+      cerr << scprintf("Double::pool: cannot reinitialize pool\n");
+      abort();
+    }
+  pool_ = p;
+}
+
+void test1(Pool*);
+void test2(Pool*);
+
+int
+main()
+{
+  const int poolsize = 4000000;
+  Pool *pool = new(malloc(poolsize)) Pool(poolsize);
+
+  Double::pool(pool);
+
+  srand48(100);
+
+  cout << "test1:" << endl;
+  test1(pool);
+  cout << "test2:" << endl;
+  test2(pool);
+
+  return 0;
+}
+
+
+void
+test1(Pool*pool)
+{
+  pool->check();
+
+  pool->print();
+  cout << endl;
+
+  Double t1(10);
+
+  Double::zapall();
+
+  pool->check();
+
+  pool->print();
+  cout << endl;
+
+  Double t2(10000);
+
+  Double::zapall();
+
+  pool->check();
+
+  Double t3(100);
+
+  Double::zapall();
+
+  pool->check();
+  
+  pool->print();
+  cout << endl;
+
+  Double::zapall();
+
+  pool->check();
+  
+  pool->print();
+  cout << endl;
+
+  t2.clear();
+
+  pool->check();
+  
+  pool->print();
+  cout << endl;
+
+  Double t4(100);
+
+  pool->check();
+
+  pool->print();
+  cout << endl;
+
+  t1.clear();
+  t4.clear();
+  t3.clear();
+
+  pool->check();
+
+  pool->print();
+  cout << endl;
+
+}
+
+void
+test2(Pool*pool)
+{
+  int i, ii;
+
+  const int npass = 10;
+  const int nd = 512;
+  Double* d[nd];
+
+  for (i=0; i<nd; i++) d[i] = 0;
+       
+  for (ii=0; ii<npass; ii++) {
+      for (i=0; i<nd;) {
+          if (mrand48() > 0) {
+              // allocate data
+              size_t size = lrand48() & 0x03ff;
+              d[i] = new Double(size);
+              cerr.flush();
+              cout << "a" << flush;
+              i++;
+            }
+          else {
+              // deallocate data
+              int loc = (int) (drand48()*i);
+              if (loc >= nd) loc = nd-1;
+              if (loc < 0) loc = 0;
+              if (d[loc]) {
+                  cerr.flush();
+                  cout << "d" << flush;
+                  delete d[loc];
+                  d[loc] = 0;
+                }
+            }
+          //pool->print();
+          //pool->check();
+          //Double::zapall();
+        }
+      for (i=0; i<nd; i++) {
+          if (d[i]) {
+              cerr.flush();
+              cout << "d" << flush;
+              delete d[i];
+              d[i] = 0;
+            }
+        }
+      cout << endl;
+      pool->print();
+      //pool->check();
+      //Double::zapall();
+    }
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/pregtime.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/pregtime.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/pregtime.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,353 @@
+//
+// pregtime.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#ifdef HAVE_CONFIG_H
+#  include <scconfig.h>
+#endif
+
+#include <iostream>
+#include <iomanip>
+
+#include <util/misc/formio.h>
+#include <util/group/pregtime.h>
+
+using namespace std;
+using namespace sc;
+
+static ClassDesc ParallelRegionTimer_cd(
+  typeid(ParallelRegionTimer),"ParallelRegionTimer",1,"public RegionTimer",
+  0, create<ParallelRegionTimer>, 0);
+
+ParallelRegionTimer::ParallelRegionTimer(const Ref<KeyVal> &keyval):
+  RegionTimer(keyval)
+{
+  msg_ = MessageGrp::get_default_messagegrp();
+}
+
+ParallelRegionTimer::ParallelRegionTimer(const Ref<MessageGrp>&msg,
+                                         const char *topname,
+                                         int cpu_time, int wall_time):
+  RegionTimer(topname, cpu_time, wall_time),
+  msg_(msg)
+{
+}
+
+ParallelRegionTimer::~ParallelRegionTimer()
+{
+}
+
+static void
+send_string(const Ref<MessageGrp>& msg, int node, const char *s)
+{
+  int l = strlen(s);
+  msg->send(node, l);
+  msg->send(node, s, l);
+}
+
+static char *
+recv_string(const Ref<MessageGrp>& msg, int node)
+{
+  int l;
+  msg->recv(node, l);
+  char *s = new char[l+1];
+  s[l] = '\0';
+  msg->recv(node, s, l);
+  return s;
+}
+
+void
+ParallelRegionTimer::send_subregions(int node, const TimedRegion *r) const
+{
+  TimedRegion *subr = r->subregions();
+
+  // rewind to the beginning
+  if (subr) { while (subr->prev()) subr = subr->prev(); }
+
+  while (subr) {
+      msg_->send(node, 1);
+      send_string(msg_, node, subr->name());
+      send_subregions(node, subr);
+      subr = subr->next();
+    };
+
+  msg_->send(node, 0);
+}
+
+void
+ParallelRegionTimer::recv_subregions(int node, TimedRegion *r) const
+{
+  int has_subregions;
+  msg_->recv(node, has_subregions);
+  while (has_subregions) {
+      char *name = recv_string(msg_, node);
+      TimedRegion *region = r->findinsubregion(name);
+      delete[] name;
+      recv_subregions(node, region);
+      msg_->recv(node, has_subregions);
+    }
+}
+
+void
+ParallelRegionTimer::all_reduce_regions() const
+{
+  Ref<MachineTopology> topology = msg_->topology();
+
+  // accumulate all the regions onto node zero
+  Ref<GlobalMsgIter> i_reduce(topology->global_msg_iter(msg_, 0));
+  for (i_reduce->backwards(); !i_reduce->done(); i_reduce->next()) {
+      if (i_reduce->send()) {
+          send_subregions(i_reduce->sendto(), top_);
+        }
+      if (i_reduce->recv()) {
+          recv_subregions(i_reduce->recvfrom(), top_);
+        }
+    }
+
+  // broadcast the regions to all the nodes
+  Ref<GlobalMsgIter> i_bcast(topology->global_msg_iter(msg_, 0));
+  for (i_bcast->forwards(); !i_bcast->done(); i_bcast->next()) {
+      if (i_bcast->send()) {
+          send_subregions(i_bcast->sendto(), top_);
+        }
+      if (i_bcast->recv()) {
+          recv_subregions(i_bcast->recvfrom(), top_);
+        }
+    }
+}
+
+void
+ParallelRegionTimer::print(ostream &o) const
+{
+  int i,j;
+
+  if (msg_->n() == 1) {
+      RegionTimer::print(o);
+      return;
+    }
+
+  update_top();
+
+  // make sure all the nodes have the same regions
+  all_reduce_regions();
+
+  int n = nregion();
+
+  double *cpu_time = 0;
+  double *wall_time = 0;
+  double *flops = 0;
+  double *min_cpu_time = 0;
+  double *min_wall_time = 0;
+  double *min_flops = 0;
+  double *max_cpu_time = 0;
+  double *max_wall_time = 0;
+  double *max_flops = 0;
+  double *avg_cpu_time = 0;
+  double *avg_wall_time = 0;
+  double *avg_flops = 0;
+  if (cpu_time_) {
+      cpu_time = new double[n];
+      get_cpu_times(cpu_time);
+      min_cpu_time = new double[n];
+      get_cpu_times(min_cpu_time);
+      max_cpu_time = new double[n];
+      get_cpu_times(max_cpu_time);
+      avg_cpu_time = new double[n];
+      get_cpu_times(avg_cpu_time);
+      msg_->max(max_cpu_time,n);
+      msg_->min(min_cpu_time,n);
+      msg_->sum(avg_cpu_time,n);
+      for (i=0; i<n; i++) {
+          avg_cpu_time[i] /= msg_->n();
+        }
+    }
+  if (wall_time_) {
+      wall_time = new double[n];
+      get_wall_times(wall_time);
+      min_wall_time = new double[n];
+      get_wall_times(min_wall_time);
+      max_wall_time = new double[n];
+      get_wall_times(max_wall_time);
+      avg_wall_time = new double[n];
+      get_wall_times(avg_wall_time);
+      msg_->max(max_wall_time,n);
+      msg_->min(min_wall_time,n);
+      msg_->sum(avg_wall_time,n);
+      for (i=0; i<n; i++) {
+          avg_wall_time[i] /= msg_->n();
+        }
+    }
+  const char *flops_name = 0;
+  if (flops_) {
+      flops= new double[n];
+      get_flops(flops);
+      if (cpu_time_) {
+        for (i=0; i<n; i++) {
+          if (fabs(cpu_time[i]) > 1.0e-10) flops[i] /= cpu_time[i]*1000000.;
+          else flops[i] = 0.0;
+          }
+        flops_name = "MFLOP/S";
+        }
+      else if (wall_time_) {
+        for (i=0; i<n; i++) {
+          if (fabs(wall_time[i]) > 1.0e-10) flops[i] /= wall_time[i]*1000000.;
+          else flops[i] = 0.0;
+          }
+        flops_name = "MFLOP/WS";
+        }
+      else {
+        for (i=0; i<n; i++) {
+          flops[i] /= 1000000.;
+          }
+        flops_name = "mflops";
+        }
+      min_flops= new double[n];
+      memcpy(min_flops, flops, sizeof(double)*n);
+      max_flops= new double[n];
+      memcpy(max_flops, flops, sizeof(double)*n);
+      avg_flops= new double[n];
+      memcpy(avg_flops, flops, sizeof(double)*n);
+      msg_->max(max_flops,n);
+      msg_->min(min_flops,n);
+      msg_->sum(avg_flops,n);
+      for (i=0; i<n; i++) {
+          avg_flops[i] /= msg_->n();
+        }
+    }
+
+  if (msg_->me() == 0) {
+      const char **names = new const char*[n];
+      get_region_names(names);
+      int *depth = new int[n];
+      get_depth(depth);
+
+      int maxwidth = 0;
+      double maxtime = 0.0;
+      for (i=0; i<n; i++) {
+          int width = strlen(names[i]) + 2 * depth[i] + 2;
+          if (width > maxwidth) maxwidth = width;
+          if (cpu_time_ && max_cpu_time[i] > maxtime)
+              maxtime = max_cpu_time[i];
+          if (wall_time_ && max_wall_time[i] > maxtime)
+              maxtime = max_wall_time[i];
+          if (flops_ && max_flops[i] > maxtime)
+              maxtime = max_flops[i];
+        }
+
+      int maxtimewidth = 4;
+      while (maxtime >= 10.0) { maxtime/=10.0; maxtimewidth++; }
+
+      o.setf(ios::right);
+
+      for (i=0; i<maxwidth; i++) o << " ";
+      if (cpu_time_) {
+          o << setw(maxtimewidth+1) << " ";
+          o << setw(maxtimewidth+1) << " CPU";
+          o << setw(maxtimewidth+1) << " ";
+        }
+      if (wall_time_) {
+          o << setw(maxtimewidth+1) << " ";
+          o << setw(maxtimewidth+1) << " Wall";
+          o << setw(maxtimewidth+1) << " ";
+        }
+      if (flops_) {
+          o << setw(maxtimewidth+1) << " ";
+          o << " " << setw(maxtimewidth+1) << flops_name;
+          o << setw(maxtimewidth+1) << " ";
+        }
+      o << endl;
+
+      for (i=0; i<maxwidth; i++) o << " ";
+      if (cpu_time_) {
+          o << setw(maxtimewidth+1) << " min";
+          o << setw(maxtimewidth+1) << " max";
+          o << setw(maxtimewidth+1) << " avg";
+        }
+      if (wall_time_) {
+          o << setw(maxtimewidth+1) << " min";
+          o << setw(maxtimewidth+1) << " max";
+          o << setw(maxtimewidth+1) << " avg";
+        }
+      if (flops_) {
+          o << setw(maxtimewidth+1) << " min";
+          o << setw(maxtimewidth+1) << " max";
+          o << setw(maxtimewidth+1) << " avg";
+        }
+      o << endl;
+
+      o.setf(ios::fixed);
+      o.precision(2);
+      for (i=0; i<n; i++) {
+          int width = strlen(names[i]) + 2 * depth[i] + 2;
+          for (j=0; j<depth[i]; j++) o << "  ";
+          o << names[i] << ": ";
+          for (j=width; j<maxwidth; j++) o << " ";
+          if (cpu_time_) {
+              o << " " << setw(maxtimewidth) << min_cpu_time[i];
+              o << " " << setw(maxtimewidth) << max_cpu_time[i];
+              o << " " << setw(maxtimewidth) << avg_cpu_time[i];
+            }                    
+          if (wall_time_) {
+              o << " " << setw(maxtimewidth) << min_wall_time[i];
+              o << " " << setw(maxtimewidth) << max_wall_time[i];
+              o << " " << setw(maxtimewidth) << avg_wall_time[i];
+            }
+          if (flops_) {
+              o << " " << setw(maxtimewidth) << min_flops[i];
+              o << " " << setw(maxtimewidth) << max_flops[i];
+              o << " " << setw(maxtimewidth) << avg_flops[i];
+            }
+          o << endl;
+        }
+
+      delete[] names;
+      delete[] depth;
+    }
+
+  delete[] cpu_time;
+  delete[] min_cpu_time;
+  delete[] max_cpu_time;
+  delete[] avg_cpu_time;
+  delete[] wall_time;
+  delete[] min_wall_time;
+  delete[] max_wall_time;
+  delete[] avg_wall_time;
+  delete[] flops;
+  delete[] min_flops;
+  delete[] max_flops;
+  delete[] avg_flops;
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/pregtime.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/pregtime.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/pregtime.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,66 @@
+//
+// pregtime.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_pregtime_h
+#define _util_group_pregtime_h
+
+#include <iostream>
+#include <util/misc/regtime.h>
+#include <util/group/message.h>
+
+namespace sc {
+
+class ParallelRegionTimer: public RegionTimer {
+  protected:
+    Ref<MessageGrp> msg_;
+
+    void send_subregions(int node, const TimedRegion *r) const;
+    void recv_subregions(int node, TimedRegion *r) const;
+    void all_reduce_regions() const;
+
+  public:
+    ParallelRegionTimer(const Ref<MessageGrp>&,
+                        const char *topname = "total",
+                        int cpu_time = 0, int wall_time = 1);
+    ParallelRegionTimer(const Ref<KeyVal>&);
+    ~ParallelRegionTimer();
+
+    void print(std::ostream& = ExEnv::out0()) const;
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/prttest.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/prttest.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/prttest.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,63 @@
+
+#include <scconfig.h>
+#include <util/group/pregtime.h>
+#include <util/misc/exenv.h>
+#if defined(HAVE_MPI)
+#  include <util/group/messmpi.h>
+#endif
+
+
+void
+dotest(const sc::Ref<sc::MessageGrp> &grp,
+       bool node_independent)
+{
+  sc::Ref<sc::RegionTimer> rt
+      = new sc::ParallelRegionTimer(grp->clone(),"prttest",1,1);
+
+  sc::Timer t1(rt,"t1_a");
+
+  if (node_independent || grp->me() == 0) {
+      sc::Timer t2(rt,"t2_a");
+      sc::Timer t3(rt,"t3_a");
+      t3.reset("t3_b");
+      t3.reset("t3_c");
+      t3.reset("t3_d");
+      t3.reset();
+      t2.reset("t2_b");
+      if (node_independent || grp->me() == 0) {
+          sc::Timer t4(rt,"t4_a");
+          t4.reset("t4_b");
+          t4.reset();
+        }
+      t2.reset("t2_c");
+      t2.reset();
+    }
+
+  t1.reset();
+
+  rt->print(sc::ExEnv::outn());
+}
+
+int
+main(int argc, char**argv)
+{
+  sc::Ref<sc::MessageGrp> grp;
+#if defined(HAVE_MPI) && defined(ALWAYS_USE_MPI)
+  grp = new sc::MPIMessageGrp(&argc, &argv);
+#endif
+  if (grp.null()) grp = sc::MessageGrp::initial_messagegrp(argc, argv);
+  if (grp.nonnull())
+      sc::MessageGrp::set_default_messagegrp(grp);
+  else
+      grp = sc::MessageGrp::get_default_messagegrp();
+
+  std::cout << grp->me() << ": using " << grp->class_name() << std::endl;
+
+  dotest(grp, true);
+  dotest(grp, false);
+
+  grp = 0;
+  sc::MessageGrp::set_default_messagegrp(grp);
+
+  return 0;
+}
Index: ThirdParty/mpqc_open/src/lib/util/group/reduce.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/reduce.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/reduce.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,633 @@
+//
+// reduce.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef HAVE_CONFIG_H
+#include <scconfig.h>
+#endif
+#include <util/group/message.h>
+
+using namespace sc;
+
+/////////////////////////////////////////////////////////////////////////
+// instantiate templates
+
+#ifdef EXPLICIT_TEMPLATE_INSTANTIATION
+template class GrpReduce<double>;
+template class GrpReduce<unsigned int>;
+template class GrpReduce<int>;
+template class GrpReduce<long>;
+template class GrpReduce<float>;
+template class GrpReduce<short>;
+template class GrpReduce<char>;
+template class GrpReduce<unsigned char>;
+template class GrpReduce<signed char>;
+
+template class GrpFunctionReduce<double>;
+template class GrpFunctionReduce<unsigned int>;
+template class GrpFunctionReduce<int>;
+template class GrpFunctionReduce<long>;
+template class GrpFunctionReduce<float>;
+template class GrpFunctionReduce<short>;
+template class GrpFunctionReduce<char>;
+template class GrpFunctionReduce<unsigned char>;
+template class GrpFunctionReduce<signed char>;
+
+template class GrpMinReduce<double>;
+template class GrpMinReduce<unsigned int>;
+template class GrpMinReduce<int>;
+template class GrpMinReduce<long>;
+template class GrpMinReduce<float>;
+template class GrpMinReduce<short>;
+template class GrpMinReduce<char>;
+template class GrpMinReduce<unsigned char>;
+template class GrpMinReduce<signed char>;
+
+template class GrpMaxReduce<double>;
+template class GrpMaxReduce<unsigned int>;
+template class GrpMaxReduce<int>;
+template class GrpMaxReduce<long>;
+template class GrpMaxReduce<float>;
+template class GrpMaxReduce<short>;
+template class GrpMaxReduce<char>;
+template class GrpMaxReduce<unsigned char>;
+template class GrpMaxReduce<signed char>;
+
+template class GrpSumReduce<double>;
+template class GrpSumReduce<unsigned int>;
+template class GrpSumReduce<int>;
+template class GrpSumReduce<long>;
+template class GrpSumReduce<float>;
+template class GrpSumReduce<short>;
+template class GrpSumReduce<char>;
+template class GrpSumReduce<unsigned char>;
+template class GrpSumReduce<signed char>;
+
+template class GrpProductReduce<double>;
+template class GrpProductReduce<unsigned int>;
+template class GrpProductReduce<int>;
+template class GrpProductReduce<long>;
+template class GrpProductReduce<float>;
+template class GrpProductReduce<short>;
+template class GrpProductReduce<char>;
+template class GrpProductReduce<unsigned char>;
+template class GrpProductReduce<signed char>;
+
+template class GrpArithmeticOrReduce<unsigned int>;
+template class GrpArithmeticOrReduce<int>;
+template class GrpArithmeticOrReduce<long>;
+template class GrpArithmeticOrReduce<short>;
+template class GrpArithmeticOrReduce<char>;
+template class GrpArithmeticOrReduce<unsigned char>;
+template class GrpArithmeticOrReduce<signed char>;
+
+template class GrpArithmeticAndReduce<unsigned int>;
+template class GrpArithmeticAndReduce<int>;
+template class GrpArithmeticAndReduce<long>;
+template class GrpArithmeticAndReduce<short>;
+template class GrpArithmeticAndReduce<char>;
+template class GrpArithmeticAndReduce<unsigned char>;
+template class GrpArithmeticAndReduce<signed char>;
+
+template class GrpArithmeticXOrReduce<unsigned int>;
+template class GrpArithmeticXOrReduce<int>;
+template class GrpArithmeticXOrReduce<long>;
+template class GrpArithmeticXOrReduce<short>;
+template class GrpArithmeticXOrReduce<char>;
+template class GrpArithmeticXOrReduce<unsigned char>;
+template class GrpArithmeticXOrReduce<signed char>;
+#endif
+
+/////////////////////////////////////////////////////////////////////////
+// sum reduction members
+
+template <class T>
+void
+do_sum(MessageGrp* grp, T* data, int n, T* tmp, int target)
+{
+  GrpSumReduce<T> gred;
+  grp->reduce(data, n, gred, tmp, target);
+}
+
+void
+MessageGrp::sum(double* data, int n, double* tmp, int target)
+{
+  do_sum(this, data, n, tmp, target);
+}
+
+void
+MessageGrp::sum(unsigned int* data, int n, unsigned int* tmp, int target)
+{
+  do_sum(this, data, n, tmp, target);
+}
+
+void
+MessageGrp::sum(int* data, int n, int* tmp, int target)
+{
+  do_sum(this, data, n, tmp, target);
+}
+
+void
+MessageGrp::sum(char* data, int n, char* tmp, int target)
+{
+  do_sum(this, data, n, tmp, target);
+}
+
+void
+MessageGrp::sum(unsigned char* data, int n, unsigned char* tmp, int target)
+{
+  do_sum(this, data, n, tmp, target);
+}
+
+void
+MessageGrp::sum(signed char* data, int n, signed char* tmp, int target)
+{
+  do_sum(this, data, n, tmp, target);
+}
+
+/////////////////////////////////////////////////////////////////////////
+// min reduction members
+
+template <class T>
+void
+do_max(MessageGrp* grp, T* data, int n, T* tmp, int target)
+{
+  GrpMaxReduce<T> gred;
+  grp->reduce(data, n, gred, tmp, target);
+}
+
+void
+MessageGrp::max(double* data, int n, double* tmp, int target)
+{
+  do_max(this, data, n, tmp, target);
+}
+
+void
+MessageGrp::max(unsigned int* data, int n, unsigned int* tmp, int target)
+{
+  do_max(this, data, n, tmp, target);
+}
+
+void
+MessageGrp::max(int* data, int n, int* tmp, int target)
+{
+  do_max(this, data, n, tmp, target);
+}
+
+void
+MessageGrp::max(char* data, int n, char* tmp, int target)
+{
+  do_max(this, data, n, tmp, target);
+}
+
+void
+MessageGrp::max(unsigned char* data, int n, unsigned char* tmp, int target)
+{
+  do_max(this, data, n, tmp, target);
+}
+
+void
+MessageGrp::max(signed char* data, int n, signed char* tmp, int target)
+{
+  do_max(this, data, n, tmp, target);
+}
+
+/////////////////////////////////////////////////////////////////////////
+// max reduction members
+
+template <class T>
+void
+do_min(MessageGrp* grp, T* data, int n, T* tmp, int target)
+{
+  GrpMinReduce<T> gred;
+  grp->reduce(data, n, gred, tmp, target);
+}
+
+void
+MessageGrp::min(double* data, int n, double* tmp, int target)
+{
+  do_min(this, data, n, tmp, target);
+}
+
+void
+MessageGrp::min(unsigned int* data, int n, unsigned int* tmp, int target)
+{
+  do_min(this, data, n, tmp, target);
+}
+
+void
+MessageGrp::min(int* data, int n, int* tmp, int target)
+{
+  do_min(this, data, n, tmp, target);
+}
+
+void
+MessageGrp::min(char* data, int n, char* tmp, int target)
+{
+  do_min(this, data, n, tmp, target);
+}
+
+void
+MessageGrp::min(unsigned char* data, int n, unsigned char* tmp, int target)
+{
+  do_min(this, data, n, tmp, target);
+}
+
+void
+MessageGrp::min(signed char* data, int n, signed char* tmp, int target)
+{
+  do_min(this, data, n, tmp, target);
+}
+
+/////////////////////////////////////////////////////////////////////////
+// generic reduction
+
+void
+MessageGrp::reduce(double* data, int n, GrpReduce<double>& red,
+                   double* scratch, int target)
+{
+  int tgop_max = gop_max_/sizeof(double);
+  if (tgop_max == 0) tgop_max = gop_max_?1:n;
+
+  int passed_scratch;
+  if (!scratch) {
+      scratch = new double[n>tgop_max?tgop_max:n];
+      passed_scratch = 0;
+    }
+  else passed_scratch = 1;
+
+  Ref<GlobalMsgIter> i(topology_->global_msg_iter(this,
+                                                    (target== -1?0:target)));
+  for (i->backwards(); !i->done(); i->next()) {
+      for (int idat=0; idat<n; idat+=tgop_max) {
+          int ndat = (idat+tgop_max>n)?(n-idat):tgop_max;
+          if (i->send()) {
+              send(i->sendto(), &data[idat], ndat);
+            }
+          if (i->recv()) {
+              recv(i->recvfrom(), scratch, ndat);
+              red.reduce(&data[idat], scratch, ndat);
+            }
+        }
+      if (n > tgop_max) sync();
+    }
+
+  if (target == -1) {
+      bcast(data, n, 0);
+    }
+
+  if (!passed_scratch) delete[] scratch;
+}
+
+void
+MessageGrp::reduce(unsigned int* data, int n, GrpReduce<unsigned int>& red,
+                   unsigned int* scratch, int target)
+{
+  int tgop_max = gop_max_/sizeof(unsigned int);
+  if (tgop_max == 0) tgop_max = gop_max_?1:n;
+
+  int passed_scratch;
+  if (!scratch) {
+      scratch = new unsigned int[n>tgop_max?tgop_max:n];
+      passed_scratch = 0;
+    }
+  else passed_scratch = 1;
+
+  Ref<GlobalMsgIter> i(topology_->global_msg_iter(this,
+                                                    (target== -1?0:target)));
+  for (i->backwards(); !i->done(); i->next()) {
+      for (int idat=0; idat<n; idat+=tgop_max) {
+          int ndat = (idat+tgop_max>n)?(n-idat):tgop_max;
+          if (i->send()) {
+              send(i->sendto(), &data[idat], ndat);
+            }
+          if (i->recv()) {
+              recv(i->recvfrom(), scratch, ndat);
+              red.reduce(&data[idat], scratch, ndat);
+            }
+        }
+      if (n > tgop_max) sync();
+    }
+
+  if (target == -1) {
+      bcast(data, n, 0);
+    }
+
+  if (!passed_scratch) delete[] scratch;
+}
+
+void
+MessageGrp::reduce(int* data, int n, GrpReduce<int>& red,
+                   int* scratch, int target)
+{
+  int tgop_max = gop_max_/sizeof(int);
+  if (tgop_max == 0) tgop_max = gop_max_?1:n;
+
+  int passed_scratch;
+  if (!scratch) {
+      scratch = new int[n>tgop_max?tgop_max:n];
+      passed_scratch = 0;
+    }
+  else passed_scratch = 1;
+
+  Ref<GlobalMsgIter> i(topology_->global_msg_iter(this,
+                                                    (target== -1?0:target)));
+  for (i->backwards(); !i->done(); i->next()) {
+      for (int idat=0; idat<n; idat+=tgop_max) {
+          int ndat = (idat+tgop_max>n)?(n-idat):tgop_max;
+          if (i->send()) {
+              send(i->sendto(), &data[idat], ndat);
+            }
+          if (i->recv()) {
+              recv(i->recvfrom(), scratch, ndat);
+              red.reduce(&data[idat], scratch, ndat);
+            }
+        }
+      if (n > tgop_max) sync();
+    }
+
+  if (target == -1) {
+      bcast(data, n, 0);
+    }
+
+  if (!passed_scratch) delete[] scratch;
+}
+
+void
+MessageGrp::reduce(char* data, int n, GrpReduce<char>& red,
+                   char* scratch, int target)
+{
+  int tgop_max = gop_max_/sizeof(char);
+  if (tgop_max == 0) tgop_max = gop_max_?1:n;
+
+  int passed_scratch;
+  if (!scratch) {
+      scratch = new char[n>tgop_max?tgop_max:n];
+      passed_scratch = 0;
+    }
+  else passed_scratch = 1;
+
+  Ref<GlobalMsgIter> i(topology_->global_msg_iter(this,
+                                                    (target== -1?0:target)));
+  for (i->backwards(); !i->done(); i->next()) {
+      for (int idat=0; idat<n; idat+=tgop_max) {
+          int ndat = (idat+tgop_max>n)?(n-idat):tgop_max;
+          if (i->send()) {
+              send(i->sendto(), &data[idat], ndat);
+            }
+          if (i->recv()) {
+              recv(i->recvfrom(), scratch, ndat);
+              red.reduce(&data[idat], scratch, ndat);
+            }
+        }
+      if (n > tgop_max) sync();
+    }
+
+  if (target == -1) {
+      bcast(data, n, 0);
+    }
+
+  if (!passed_scratch) delete[] scratch;
+}
+
+void
+MessageGrp::reduce(unsigned char* data, int n, GrpReduce<unsigned char>& red,
+                   unsigned char* scratch, int target)
+{
+  int tgop_max = gop_max_/sizeof(unsigned char);
+  if (tgop_max == 0) tgop_max = gop_max_?1:n;
+
+  int passed_scratch;
+  if (!scratch) {
+      scratch = new unsigned char[n>tgop_max?tgop_max:n];
+      passed_scratch = 0;
+    }
+  else passed_scratch = 1;
+
+  Ref<GlobalMsgIter> i(topology_->global_msg_iter(this,
+                                                    (target== -1?0:target)));
+  for (i->backwards(); !i->done(); i->next()) {
+      for (int idat=0; idat<n; idat+=tgop_max) {
+          int ndat = (idat+tgop_max>n)?(n-idat):tgop_max;
+          if (i->send()) {
+              send(i->sendto(), &data[idat], ndat);
+            }
+          if (i->recv()) {
+              recv(i->recvfrom(), scratch, ndat);
+              red.reduce(&data[idat], scratch, ndat);
+            }
+        }
+      if (n > tgop_max) sync();
+    }
+
+  if (target == -1) {
+      bcast(data, n, 0);
+    }
+
+  if (!passed_scratch) delete[] scratch;
+}
+
+void
+MessageGrp::reduce(signed char* data, int n, GrpReduce<signed char>& red,
+                   signed char* scratch, int target)
+{
+  int tgop_max = gop_max_/sizeof(signed char);
+  if (tgop_max == 0) tgop_max = gop_max_?1:n;
+
+  int passed_scratch;
+  if (!scratch) {
+      scratch = new signed char[n>tgop_max?tgop_max:n];
+      passed_scratch = 0;
+    }
+  else passed_scratch = 1;
+
+  Ref<GlobalMsgIter> i(topology_->global_msg_iter(this,
+                                                    (target== -1?0:target)));
+  for (i->backwards(); !i->done(); i->next()) {
+      for (int idat=0; idat<n; idat+=tgop_max) {
+          int ndat = (idat+tgop_max>n)?(n-idat):tgop_max;
+          if (i->send()) {
+              send(i->sendto(), &data[idat], ndat);
+            }
+          if (i->recv()) {
+              recv(i->recvfrom(), scratch, ndat);
+              red.reduce(&data[idat], scratch, ndat);
+            }
+        }
+      if (n > tgop_max) sync();
+    }
+
+  if (target == -1) {
+      bcast(data, n, 0);
+    }
+
+  if (!passed_scratch) delete[] scratch;
+}
+
+void
+MessageGrp::reduce(short* data, int n, GrpReduce<short>& red,
+                   short* scratch, int target)
+{
+  int tgop_max = gop_max_/sizeof(short);
+  if (tgop_max == 0) tgop_max = gop_max_?1:n;
+
+  int passed_scratch;
+  if (!scratch) {
+      scratch = new short[n>tgop_max?tgop_max:n];
+      passed_scratch = 0;
+    }
+  else passed_scratch = 1;
+
+  Ref<GlobalMsgIter> i(topology_->global_msg_iter(this,
+                                                    (target== -1?0:target)));
+  for (i->backwards(); !i->done(); i->next()) {
+      for (int idat=0; idat<n; idat+=tgop_max) {
+          int ndat = (idat+tgop_max>n)?(n-idat):tgop_max;
+          if (i->send()) {
+              send(i->sendto(), &data[idat], ndat);
+            }
+          if (i->recv()) {
+              recv(i->recvfrom(), scratch, ndat);
+              red.reduce(&data[idat], scratch, ndat);
+            }
+        }
+      if (n > tgop_max) sync();
+    }
+
+  if (target == -1) {
+      bcast(data, n, 0);
+    }
+
+  if (!passed_scratch) delete[] scratch;
+}
+
+void
+MessageGrp::reduce(float* data, int n, GrpReduce<float>& red,
+                   float* scratch, int target)
+{
+  int tgop_max = gop_max_/sizeof(float);
+  if (tgop_max == 0) tgop_max = gop_max_?1:n;
+
+  int passed_scratch;
+  if (!scratch) {
+      scratch = new float[n>tgop_max?tgop_max:n];
+      passed_scratch = 0;
+    }
+  else passed_scratch = 1;
+
+  Ref<GlobalMsgIter> i(topology_->global_msg_iter(this,
+                                                    (target== -1?0:target)));
+  for (i->backwards(); !i->done(); i->next()) {
+      for (int idat=0; idat<n; idat+=tgop_max) {
+          int ndat = (idat+tgop_max>n)?(n-idat):tgop_max;
+          if (i->send()) {
+              send(i->sendto(), &data[idat], ndat);
+            }
+          if (i->recv()) {
+              recv(i->recvfrom(), scratch, ndat);
+              red.reduce(&data[idat], scratch, ndat);
+            }
+        }
+      if (n > tgop_max) sync();
+    }
+
+  if (target == -1) {
+      bcast(data, n, 0);
+    }
+
+  if (!passed_scratch) delete[] scratch;
+}
+
+void
+MessageGrp::reduce(long* data, int n, GrpReduce<long>& red,
+                   long* scratch, int target)
+{
+  int tgop_max = gop_max_/sizeof(long);
+  if (tgop_max == 0) tgop_max = gop_max_?1:n;
+
+  int passed_scratch;
+  if (!scratch) {
+      scratch = new long[n>tgop_max?tgop_max:n];
+      passed_scratch = 0;
+    }
+  else passed_scratch = 1;
+
+  Ref<GlobalMsgIter> i(topology_->global_msg_iter(this,
+                                                    (target== -1?0:target)));
+  for (i->backwards(); !i->done(); i->next()) {
+      for (int idat=0; idat<n; idat+=tgop_max) {
+          int ndat = (idat+tgop_max>n)?(n-idat):tgop_max;
+          if (i->send()) {
+              send(i->sendto(), &data[idat], ndat);
+            }
+          if (i->recv()) {
+              recv(i->recvfrom(), scratch, ndat);
+              red.reduce(&data[idat], scratch, ndat);
+            }
+        }
+      if (n > tgop_max) sync();
+    }
+
+  if (target == -1) {
+      bcast(data, n, 0);
+    }
+
+  if (!passed_scratch) delete[] scratch;
+}
+
+#ifdef EXPLICIT_TEMPLATE_INSTANTIATION
+#define INSTANTIATE_DO_X(func,type) \
+    template void func(MessageGrp*, type *, int, type *, int)
+
+INSTANTIATE_DO_X(do_sum,unsigned int);
+INSTANTIATE_DO_X(do_sum,int);
+INSTANTIATE_DO_X(do_sum,double);
+INSTANTIATE_DO_X(do_sum,char);
+INSTANTIATE_DO_X(do_sum,unsigned char);
+INSTANTIATE_DO_X(do_sum,signed char);
+
+INSTANTIATE_DO_X(do_max,unsigned int);
+INSTANTIATE_DO_X(do_max,int);
+INSTANTIATE_DO_X(do_max,double);
+INSTANTIATE_DO_X(do_max,char);
+INSTANTIATE_DO_X(do_max,unsigned char);
+INSTANTIATE_DO_X(do_max,signed char);
+
+INSTANTIATE_DO_X(do_min,unsigned int);
+INSTANTIATE_DO_X(do_min,int);
+INSTANTIATE_DO_X(do_min,double);
+INSTANTIATE_DO_X(do_min,char);
+INSTANTIATE_DO_X(do_min,unsigned char);
+INSTANTIATE_DO_X(do_min,signed char);
+
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/rnglock.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/rnglock.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/rnglock.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,344 @@
+//
+// rnglock.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#include <new>
+
+#include <util/misc/formio.h>
+#include <util/group/rnglock.h>
+#include <util/group/pool.h>
+
+using namespace std;
+using namespace sc;
+
+#define CHECK 0
+
+namespace sc {
+
+/////////////////////////////////////////////////////////////////////
+// RangeLockItem members
+
+void *
+RangeLockItem::operator new(size_t size, Pool * pool)
+{
+  if (pool) return pool->allocate(size);
+  else return ::operator new(size);
+}
+
+void
+RangeLockItem::operator delete(void* r, Pool *pool)
+{
+ if (pool) pool->release(r);
+ else ::operator delete(r);
+}
+
+/////////////////////////////////////////////////////////////////////
+// Utility classes
+
+class RangeLockValOp {
+  public:
+    virtual ~RangeLockValOp() {};
+    virtual void op(int&) = 0;
+};
+
+class RangeLockValSum: public RangeLockValOp {
+  private:
+    int delta;
+  public:
+    RangeLockValSum(int d): delta(d) {}
+    void op(int &i) { i += delta; }
+};
+
+class RangeLockValSet: public RangeLockValOp {
+  private:
+    int val;
+  public:
+    RangeLockValSet(int v): val(v) {}
+    void op(int &i) { i = val; }
+};
+
+/////////////////////////////////////////////////////////////////////
+// Members of RangeLock
+
+RangeLock::RangeLock(Pool *pool)
+{
+  pool_ = pool;
+  root_ = 0;
+}
+
+RangeLock::~RangeLock()
+{
+  for (RangeLockItem *i = root_; i;) {
+      RangeLockItem *next = i->next;
+      RangeLockItem::operator delete(i, pool_);
+      i = next;
+    }
+}
+
+int
+RangeLock::lockvalue(int loc)
+{
+  for (RangeLockItem *i = root_; i; i = i->next) {
+      if (loc >= i->start && loc < i->fence) {
+          return i->value;
+        }
+      if (loc < i->fence) break;
+    }
+  return 0;
+}
+
+int
+RangeLock::checkeq(int start, int fence, int value)
+{
+  split_ranges(start, fence);
+  for (RangeLockItem *i = root_; i; i = i->next) {
+      if (start >= i->start && start < i->fence
+          || fence > i->start && fence <= i->fence
+          || start < i->start && fence > i->fence) {
+          if (value != i->value) return 0;
+        }
+      if (fence < i->fence) break;
+    }
+  return 1;
+}
+
+int
+RangeLock::checkgr(int start, int fence, int value)
+{
+  split_ranges(start, fence);
+  for (RangeLockItem *i = root_; i; i = i->next) {
+      if (start >= i->start && start < i->fence
+          || fence > i->start && fence <= i->fence
+          || start < i->start && fence > i->fence) {
+          if (i->value <= value) return 0;
+        }
+      if (fence < i->fence) break;
+    }
+  return 1;
+}
+
+void
+RangeLock::check()
+{
+  for (RangeLockItem* i = root_; i; i = i->next) {
+      int bad = 0;
+      if (i->next && i->next->prev != i) {
+          ExEnv::errn() << scprintf("i->next->prev bad\n");
+          bad = 1;
+        }
+      if (i->prev && i->prev->next != i) {
+          ExEnv::errn() << scprintf("i->prev->next bad\n");
+          bad = 1;
+        }
+      if (i->start >= i->fence) {
+          ExEnv::errn() << scprintf("start >= fence\n");
+          bad = 1;
+        }
+      if (i->next && i->fence > i->next->start) {
+          ExEnv::errn() << scprintf("fence > next start\n");
+          bad = 1;
+        }
+#if VERBOSE
+      ExEnv::outn()
+          << scprintf("i = 0x%08x, n = 0x%08x, p = 0x%08x, [%3d, %3d), %5d\n",
+                      i, i->next, i->prev, i->start, i->fence, i->value);
+#endif
+      if (bad) abort();
+    }
+}
+
+void
+RangeLock::split_ranges(int start, int fence)
+{
+  if (root_ == 0) {
+      // no blocks are allocted yet, initialize one and return
+      root_ = new(pool_) RangeLockItem(0, 0, start, fence, 0);
+      return;
+    }
+
+  RangeLockItem *i;
+  for (i = root_; i; i=i->next) {
+      if (start > i->start && start < i->fence) {
+          // start is in the middle of this block, split it
+          RangeLockItem *t = new(pool_) RangeLockItem(i, i->next,
+                                                      start, i->fence,
+                                                      i->value);
+          i->fence = start;
+          i->next = t;
+          if (t->next) t->next->prev = t;
+          i = t;
+          break;
+        }
+      else if (start < i->start && fence <= i->start) {
+          // start and end are before this block, insert it and return
+          RangeLockItem *t = new(pool_) RangeLockItem(i->prev, i,
+                                                      start, fence, 0);
+          i->prev = t;
+          if (t->prev) t->prev->next = t;
+          else root_ = t;
+          return;
+        }
+      else if (start < i->start) {
+          // start is before this block, fill in the gap
+          RangeLockItem *t = new(pool_) RangeLockItem(i->prev, i,
+                                                      start, i->start, 0);
+          i->prev = t;
+          if (t->prev) t->prev->next = t;
+          else root_ = t;
+          break;
+        }
+      else if (start == i->start) {
+          // start coincides with this block's start
+          break;
+        }
+      else if (i->next == 0) {
+          // start is after the last block, make the block and return
+          RangeLockItem *t = new(pool_) RangeLockItem(i, 0, start, fence, 0);
+          i->next = t;
+          return;
+        }
+      // otherwise start is after this block, continue
+    }
+
+  for (; i; i=i->next) {
+      if (fence > i->start && i->prev && i->prev->fence < i->start) {
+          // fence is after this block and there is a gap before
+          RangeLockItem *t = new(pool_) RangeLockItem(i->prev, i,
+                                                      i->prev->fence, i->start,
+                                                      0);
+          i->prev = t;
+          if (t->prev) t->prev->next = t;
+          else root_ = t;
+          i = t;
+        }
+      else if (fence > i->start && fence < i->fence) {
+          // fence is in the middle of this block, split it and return
+          RangeLockItem *t = new(pool_) RangeLockItem(i->prev, i,
+                                                      i->start, fence,
+                                                      i->value);
+          i->start = fence;
+          i->prev = t;
+          if (t->prev) t->prev->next = t;
+          else root_ = t;
+          return;
+        }
+      else if (fence > i->fence && !i->next) {
+          // fence is after this block and no blocks follow
+          RangeLockItem *t = new(pool_) RangeLockItem(i, 0,
+                                                      i->fence, fence, 0);
+          i->next = t;
+          return;
+        }
+      else if (fence <= i->start) {
+          // fence is before this block, fill in the gap
+          RangeLockItem *p = i->prev;
+          RangeLockItem *t = new(pool_) RangeLockItem(p, i,
+                                                      p->fence, fence, 0);
+          p->next = t;
+          i->prev = t;
+          return;
+        }
+      else if (fence == i->fence) {
+          // fence coincides with this block's fence
+          return;
+        }
+      // otherwise fence is after this block, continue
+    }
+
+  ExEnv::errn() << scprintf("RangeLock::split_ranges(): got to end\n");
+  abort();
+}
+
+void
+RangeLock::do_valop(RangeLockValOp& op, int start, int fence)
+{
+  if (start == fence) return;
+
+  split_ranges(start, fence);
+
+#if CHECK
+  check();
+#endif
+
+  for (RangeLockItem *i = root_; i; i = i->next) {
+      if (start == i->start
+          || fence == i->fence
+          || start < i->start && fence > i->fence) {
+          op.op(i->value);
+        }
+      if (fence < i->fence) break;
+    }
+}
+
+void
+RangeLock::sum(int start, int fence, int delta)
+{
+  RangeLockValSum sum(delta);
+  do_valop(sum, start, fence);
+}
+
+void
+RangeLock::increment(int start, int fence)
+{
+  RangeLockValSum sum(1);
+  do_valop(sum, start, fence);
+}
+
+void
+RangeLock::decrement(int start, int fence)
+{
+  RangeLockValSum sum(-1);
+  do_valop(sum, start, fence);
+}
+
+void
+RangeLock::set(int start, int fence, int value)
+{
+  RangeLockValSet set(value);
+  do_valop(set, start, fence);
+}
+
+void
+RangeLock::print(ostream &o) const
+{
+  for (RangeLockItem *i = root_; i; i = i->next) {
+      o << scprintf("  RangeLockItem: [%5d, %5d): %4d\n",
+              i->start, i->fence, i->value);
+    }
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+}
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/rnglock.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/rnglock.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/rnglock.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,93 @@
+//
+// rnglock.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#ifndef _util_group_rnglock_h
+#define _util_group_rnglock_h
+
+#include <iostream>
+
+#include <util/misc/exenv.h>
+
+namespace sc {
+
+class Pool;
+
+class RangeLockItem {
+  public:
+    RangeLockItem *prev;
+    RangeLockItem *next;
+    int start;
+    int fence;
+    int value;
+    RangeLockItem(RangeLockItem *p, RangeLockItem *n, int s, int f, int v):
+      prev(p), next(n), start(s), fence(f), value(v) {}
+    ~RangeLockItem() {};
+
+    static void *operator new(size_t, Pool *);
+    static void operator delete(void *, Pool *);
+};
+
+class RangeLockValOp;
+class RangeLock {
+  private:
+    RangeLockItem *root_;
+    Pool *pool_;
+
+    void split_ranges(int start, int fence);
+    void do_valop(RangeLockValOp&, int start, int fence);
+  public:
+    RangeLock(Pool *pool = 0);
+    ~RangeLock();
+
+    void increment(int start, int fence);
+    void decrement(int start, int fence);
+    void set(int start, int fence, int value);
+    void sum(int start, int fence, int delta);
+
+    // check for anything within a range to be equal to a value
+    int checkeq(int start, int fence, int value);
+    // check for anything within a range to be greater than a value
+    int checkgr(int start, int fence, int value);
+
+    void check();
+    void print(std::ostream &o = ExEnv::out0()) const;
+
+    int lockvalue(int i);
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/rnglocktest.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/rnglocktest.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/rnglocktest.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,83 @@
+//
+// rnglocktest.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#define TEST 1
+#define VERBOSE 0
+
+#include <math.h>
+#include <stdlib.h>
+#include <util/misc/formio.h>
+#include <util/group/rnglock.h>
+
+using namespace std;
+using namespace sc;
+
+main()
+{
+  const int size = 10000;
+  int bin[size];
+  int i;
+
+  RangeLock lock;
+
+  for (i=0; i<size; i++) bin[i] = 0;
+
+  for (i=0; i<10000; i++) {
+      int start = random()%size;
+      int length = random()%30;
+      if (length == 0) length++;
+      int fence = start + length;
+      if (fence > size) fence = size;
+      int val = random()%2 ? -1:1;
+      val = 1;
+#if VERBOSE
+      ExEnv::out0() << scprintf("adding block %d: start = %d, fence = %d, val = %d\n",
+                       i, start, fence, val);
+#endif
+      lock.sum(start, fence, val);
+      for (int j=start; j<fence; j++) {
+          bin[j] += val;
+        }
+    }
+
+  ExEnv::out0() << "printing nonzero deltas" << endl;
+
+  for (i=0; i<size; i++) {
+      int delta = bin[i] - lock.lockvalue(i);
+      if (delta) ExEnv::out0() << scprintf(" %5d: %8d (%8d %8d)\n", i, delta,
+                                  bin[i], lock.lockvalue(i));
+    }
+
+  return 0;
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/thpthd.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/thpthd.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/thpthd.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,278 @@
+//
+// thpthd.cc
+//
+// Copyright (C) 1997 Limit Point Systems, Inc.
+//
+// Author: Edward Seidl <seidl@janed.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#ifdef HAVE_CONFIG_H
+#include <scconfig.h>
+#endif
+
+#if HAVE_PTHREAD_H
+#include <pthread.h>
+#endif
+
+#include <string.h>
+
+#include <util/keyval/keyval.h>
+#include <util/group/thpthd.h>
+#include <util/misc/formio.h>
+
+using namespace std;
+using namespace sc;
+
+namespace sc {
+
+/////////////////////////////////////////////////////////////////////////////
+// PthreadThreadLock class
+
+class PthreadThreadLock : public ThreadLock {
+  private:
+    pthread_mutex_t mutex_;
+    pthread_mutexattr_t attr_;
+    
+  public:
+    PthreadThreadLock() {
+      pthread_mutexattr_init(&attr_);
+//#if defined(PTHREAD_MUTEX_FAST_NP)
+//      pthread_mutexattr_setkind_np(&attr_, PTHREAD_MUTEX_FAST_NP);
+//#elif defined(MUTEX_FAST_NP)
+//      pthread_mutexattr_setkind_np(&attr_, MUTEX_FAST_NP);
+//#endif
+      pthread_mutex_init(&mutex_, &attr_);
+    }
+
+    ~PthreadThreadLock() {
+      pthread_mutexattr_destroy(&attr_);
+      pthread_mutex_destroy(&mutex_);
+    }
+
+    void lock() { pthread_mutex_lock(&mutex_); }
+    void unlock() { pthread_mutex_unlock(&mutex_); }
+};
+
+/////////////////////////////////////////////////////////////////////////////
+// PthreadThreadGrp members
+
+static ClassDesc PthreadThreadGrp_cd(
+  typeid(PthreadThreadGrp),"PthreadThreadGrp",1,"public ThreadGrp",
+  0, create<PthreadThreadGrp>, 0);
+
+PthreadThreadGrp::PthreadThreadGrp()
+  : ThreadGrp()
+{
+  pthreads_ = new pthread_t[nthread_];
+  init_attr();
+}
+
+
+PthreadThreadGrp::PthreadThreadGrp(const PthreadThreadGrp &tg,int nthread):
+  ThreadGrp(tg, nthread)
+{
+  pthreads_ = new pthread_t[nthread_];
+  init_attr();
+}
+
+PthreadThreadGrp::PthreadThreadGrp(const Ref<KeyVal>& keyval)
+  : ThreadGrp(keyval)
+{
+  pthreads_ = new pthread_t[nthread_];
+  init_attr();
+}
+
+PthreadThreadGrp::~PthreadThreadGrp()
+{
+  if (pthreads_) {
+    delete[] pthreads_;
+    pthreads_ = 0;
+    delete[] attr_;
+ }
+//  delete attr_;
+}
+
+void
+PthreadThreadGrp::init_attr()
+{
+  attr_ = new pthread_attr_t[nthread_];
+
+  for (int i=0; i<nthread_; i++) {
+    pthread_attr_init(&attr_[i]);
+#if defined(PTHREAD_CREATE_UNDETACHED)
+    pthread_attr_setdetachstate(&attr_[i], PTHREAD_CREATE_UNDETACHED);
+#elif defined(PTHREAD_CREATE_JOINABLE)
+    pthread_attr_setdetachstate(&attr_[i], PTHREAD_CREATE_JOINABLE);
+#endif
+#ifdef HAVE_PTHREAD_ATTR_GETSTACKSIZE
+    size_t defstacksize;
+    pthread_attr_getstacksize(&attr_[i], &defstacksize);
+#elif HAVE_PTHREAD_ATTR_SETSTACKSIZE
+    size_t defstacksize = 1;
+#endif
+#ifdef HAVE_PTHREAD_ATTR_SETSTACKSIZE
+    size_t minstacksize = 2097152;
+    if (defstacksize < minstacksize) {
+      pthread_attr_setstacksize(attr_[i], minstacksize);
+    }
+#endif
+  }
+}
+
+void PthreadThreadGrp::add_thread(int ithread, Thread* t, int priority)
+{
+  if (ithread >= nthread_) {
+    ExEnv::err0() << indent
+                 << "PthreadThreadGrp::add_thread(int, Thread*, int, int): trying to"
+                 << "add too many threads" << endl;
+  }
+  else {
+    threads_[ithread] = t;
+    //init_priority(ithread, priority);
+  }
+  
+}
+
+#if defined(HAVE_SCHED_GET_PRIORITY_MAX) \
+   && defined(HAVE_SCHED_GET_PRIORITY_MAX) \
+   && defined(HAVE_PTHREAD_ATTR_SETSCOPE) \
+   && defined(HAVE_PTHREAD_ATTR_SETSCHEDPARAM) \
+   && defined(HAVE_PTHREAD_ATTR_SETINHERITSCHED) \
+   && defined(HAVE_PTHREAD_ATTR_SETSCHEDPOLICY)
+#define THREAD_PRIORITY_CAN_BE_SET
+#else
+#undef THREAD_PRIORITY_CAN_BE_SET
+#endif
+
+void PthreadThreadGrp::init_priority(int ithread, int priority)
+{
+#ifdef THREAD_PRIORITY_CAN_BE_SET
+  struct sched_param param, low_param, high_param;
+  int rc, selected_sched, set_params;
+
+  set_params=0;
+  
+  // Check priority settings for various schedulers and select which to use
+  selected_sched=-1;
+
+#ifdef SCHED_OTHER
+  low_param.sched_priority = sched_get_priority_min(SCHED_OTHER);
+  high_param.sched_priority = sched_get_priority_max(SCHED_OTHER);
+  if (high_param.sched_priority > low_param.sched_priority) {
+    selected_sched = SCHED_OTHER;
+    set_params=1;
+  }
+#endif // SCHED_OTHER
+#ifdef SCHED_RR
+  if (!set_params) {
+    low_param.sched_priority = sched_get_priority_min(SCHED_RR);
+    high_param.sched_priority = sched_get_priority_max(SCHED_RR);
+    if (high_param.sched_priority > low_param.sched_priority) {
+      selected_sched=SCHED_RR; set_params=1;
+    }
+  }
+#endif // SCHED_RR
+#ifdef SCHED_FIFO
+  if (!set_params) {
+    low_param.sched_priority = sched_get_priority_min(SCHED_FIFO);
+    high_param.sched_priority = sched_get_priority_max(SCHED_FIFO);
+    if (high_param.sched_priority > low_param.sched_priority) {
+      selected_sched=SCHED_FIFO; set_params=1;
+    }
+  }
+#endif // SCHED_FIFO
+
+#ifdef PTHREAD_SCOPE_SYSTEM
+  pthread_attr_setscope(&attr_[ithread],PTHREAD_SCOPE_SYSTEM);
+#endif
+  if (set_params) {  
+    pthread_attr_setinheritsched(&attr_[ithread],PTHREAD_EXPLICIT_SCHED);
+    pthread_attr_setschedpolicy(&attr_[ithread], selected_sched);
+    param.sched_priority = ( sched_get_priority_min(selected_sched) + priority );
+    pthread_attr_setschedparam(&attr_[ithread],&param);
+  }
+#endif // THREAD_PRIORITY_CAN_BE_SET
+}
+int
+PthreadThreadGrp::start_threads()
+{
+  for (int i=1; i < nthread_; i++) {
+    if (threads_[i]) {
+      int res = pthread_create(&pthreads_[i], &attr_[i],
+                               Thread__run_Thread_run,
+                               (void*) threads_[i]);
+      if (res) {
+        ExEnv::errn() << indent << "pthread_create failed" << endl;
+        ExEnv::errn() << "error: " << res << ": " << strerror(res) << endl;
+        return -1;
+      }
+    }
+  }
+  if (threads_[0]) threads_[0]->run();
+
+  return 0;
+}
+
+int
+PthreadThreadGrp::wait_threads()
+{
+  for (int i=1; i < nthread_; i++) {
+    void *tn;
+    if (threads_[i]) {
+      int rc = pthread_join(pthreads_[i], (void**)&tn);
+      if (rc) {
+        ExEnv::errn()
+          << "PthreadThreadGrp::wait_threads(): error joining thread"
+          << endl;
+        ExEnv::errn() << "error: " << rc << ": " << strerror(rc) << endl;
+        abort();
+      }
+    }
+  }
+    
+  return 0;
+}
+
+Ref<ThreadLock>
+PthreadThreadGrp::new_lock()
+{
+  return new PthreadThreadLock;
+}
+
+ThreadGrp*
+PthreadThreadGrp::clone(int nthread)
+{
+  return new PthreadThreadGrp(*this,nthread);
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+}
+
+// Local Variables:
+// mode: c++
+// c-file-style: "ETS"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/thpthd.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/thpthd.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/thpthd.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,72 @@
+//
+// thpthd.h
+//
+// Copyright (C) 1997 Limit Point Systems, Inc.
+//
+// Author: Edward Seidl <seidl@janed.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_thpthd_h
+#define _util_group_thpthd_h
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#include <pthread.h>
+#include <util/group/thread.h>
+
+namespace sc {
+
+/** The PthreadThreadGrp class privides a concrete thread group
+    appropriate for an environment where pthreads is available. */
+class PthreadThreadGrp: public ThreadGrp {
+  private:
+    pthread_t *pthreads_;
+    pthread_attr_t *attr_;
+
+    void init_attr();
+    void init_priority(int, int);
+  public:
+    PthreadThreadGrp();
+    PthreadThreadGrp(const PthreadThreadGrp&, int nthread = -1);
+    PthreadThreadGrp(const Ref<KeyVal>&);
+    ~PthreadThreadGrp();
+
+    int start_threads();
+    int wait_threads();
+    void add_thread(int i, Thread* t) { ThreadGrp::add_thread(i,t); }
+    void add_thread(int i, Thread* t, int priority);
+    
+    Ref<ThreadLock> new_lock();
+
+    ThreadGrp* clone(int nthread = -1);
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "ETS"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/thpuma.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/thpuma.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/thpuma.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,122 @@
+//
+// thpuma.cc
+//
+// Copyright (C) 1997 Limit Point Systems, Inc.
+//
+// Author: Edward Seidl <seidl@janed.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+extern "C" {
+#include <userp.h>
+#include <lock.h>
+}
+
+#include <util/keyval/keyval.h>
+#include <util/group/thpuma.h>
+#include <util/misc/formio.h>
+
+using namespace sc;
+
+/////////////////////////////////////////////////////////////////////////////
+// PumaThreadLock class
+
+class PumaThreadLock : public ThreadLock {
+  private:
+    volatile int lock_;
+    
+  public:
+    PumaThreadLock() : lock_(0) {}
+    ~PumaThreadLock() {}
+
+    void lock() { set_lock(&lock_); }
+    void unlock() { clr_lock(&lock_); }
+};
+
+/////////////////////////////////////////////////////////////////////////////
+// PumaThreadGrp members
+
+static ClassDesc PumaThreadGrp_cd(
+  typeid(PumaThreadGrp),"PumaThreadGrp",1,"public ThreadGrp",
+  0, create<PumaThreadGrp>, 0);
+
+PumaThreadGrp::PumaThreadGrp()
+  : ThreadGrp()
+{
+}
+
+PumaThreadGrp::PumaThreadGrp(const Ref<KeyVal>& keyval)
+  : ThreadGrp(keyval)
+{
+  if (nthread_ > 2) {
+    delete[] threads_;
+    nthread_ = 2;
+    threads_ = new Thread*[nthread_];
+  }
+}
+
+PumaThreadGrp::~PumaThreadGrp()
+{
+}
+
+static void
+run_Thread_run(void *thread)
+{
+  Thread::run_Thread_run(thread);
+}
+
+int
+PumaThreadGrp::start_threads()
+{
+  flag_=0;
+  if (nthread_ > 1 && threads_[1])
+    cop(run_Thread_run, &flag_, (void*)threads_[1]);
+
+  if (threads_[0]) threads_[0]->run();
+  
+  return 0;
+}
+
+int
+PumaThreadGrp::wait_threads()
+{
+  if (nthread_ > 1)
+    while (!flag_);
+    
+  return 0;
+}
+
+Ref<ThreadLock>
+PumaThreadGrp::new_lock()
+{
+  return new PumaThreadLock;
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "ETS"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/thpuma.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/thpuma.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/thpuma.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,64 @@
+//
+// thpuma.h
+//
+// Copyright (C) 1997 Limit Point Systems, Inc.
+//
+// Author: Edward Seidl <seidl@janed.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_thpuma_h
+#define _util_group_thpuma_h
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#include <pthread.h>
+#include <util/group/thread.h>
+
+namespace sc {
+  
+/** The PumaThreadGrp class privides a concrete thread group
+    appropriate for the intel teraflops machine.
+*/
+class PumaThreadGrp: public ThreadGrp {
+  private:
+    volatile unsigned int flag_;
+    
+  public:
+    PumaThreadGrp();
+    PumaThreadGrp(const Ref<KeyVal>&);
+    ~PumaThreadGrp();
+
+    int start_threads();
+    int wait_threads();
+    Ref<ThreadLock> new_lock();
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "ETS"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/thread.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/thread.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/thread.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,326 @@
+//
+// thread.cc
+//
+// Copyright (C) 1997 Limit Point Systems, Inc.
+//
+// Author: Edward Seidl <seidl@janed.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUC__
+#pragma implementation
+#endif
+
+#include <util/keyval/keyval.h>
+#include <util/group/thread.h>
+#include <util/misc/formio.h>
+#include <util/misc/exenv.h>
+
+// debug includes
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <scconfig.h>
+#ifdef HAVE_PTHREAD
+#  include <util/group/thpthd.h>
+#endif
+
+using namespace std;
+using namespace sc;
+
+// This has C linkage.
+void *
+Thread__run_Thread_run(void* vth)
+{
+  return Thread::run_Thread_run(vth);
+}
+
+namespace sc {
+
+/////////////////////////////////////////////////////////////////////////////
+
+ThreadLock::ThreadLock()
+{
+}
+
+ThreadLock::~ThreadLock()
+{
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+Thread::Thread()
+{
+}
+
+Thread::~Thread()
+{
+}
+
+void *
+Thread::run_Thread_run(void* vth)
+{
+  if (vth) ((Thread*)vth)->run();
+  return 0;
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// ThreadGrp members
+
+static ClassDesc ThreadGrp_cd(typeid(ThreadGrp),"ThreadGrp",1,
+                              "public DescribedClass");
+
+ThreadGrp::ThreadGrp() : threads_(0),  nthread_(1)
+{
+  threads_ = new Thread*[nthread_];
+  for (int i = 0; i<nthread_; i++) threads_[i] = 0;
+}
+
+ThreadGrp::ThreadGrp(const ThreadGrp &tg, int nthread)
+{
+  if (nthread == -1) nthread_ = tg.nthread_;
+  else nthread_ = nthread;
+  threads_ = new Thread*[nthread_];
+  for (int i = 0; i<nthread_; i++) threads_[i] = 0;
+}
+
+ThreadGrp::ThreadGrp(const Ref<KeyVal>& keyval)
+{
+  int defaultnum = ExEnv::nproc();
+  if (defaultnum == 0) defaultnum = 1;
+  KeyValValueint num(defaultnum);
+  nthread_ = keyval->intvalue("num_threads",num);
+
+  threads_ = new Thread*[nthread_];
+  for (int i=0; i<nthread_; i++) threads_[i] = 0;
+}
+
+ThreadGrp::~ThreadGrp()
+{
+  if (nthread_) {
+    delete[] threads_;
+    nthread_=0;
+    threads_=0;
+  }
+}
+
+void
+ThreadGrp::delete_threads()
+{
+  for (int i=0; i<nthread_; i++) {
+    delete threads_[i];
+    threads_[i] = 0;
+  }
+}
+
+void
+ThreadGrp::add_thread(int i, Thread*t)
+{
+  if (i >= nthread_) {
+    ExEnv::err0() << indent
+         << "ThreadGrp::add_thread: trying to add too many threads"
+         << endl;
+  } else {
+    threads_[i] = t;
+  }
+}
+
+void
+ThreadGrp::add_thread(int i, Thread*t, int priority)
+{
+  add_thread(i,t);
+}
+
+static Ref<ThreadGrp> default_threadgrp;
+
+void
+ThreadGrp::set_default_threadgrp(const Ref<ThreadGrp>& grp)
+{
+  default_threadgrp = grp;
+}
+
+ThreadGrp*
+ThreadGrp::get_default_threadgrp()
+{
+  if (default_threadgrp.null()) {
+#ifdef HAVE_PTHREAD
+    default_threadgrp = new PthreadThreadGrp;
+#else
+    default_threadgrp = new ProcThreadGrp;
+#endif
+  }
+
+  return default_threadgrp;
+}
+
+ThreadGrp*
+ThreadGrp::initial_threadgrp(int& argc, char ** argv)
+{
+  ThreadGrp *grp = 0;
+  char * keyval_string = 0;
+  
+  // see if a thread group is given on the command line
+  if (argc && argv) {
+    for (int i=0; i < argc; i++) {
+      if (argv[i] && !strcmp(argv[i], "-threadgrp")) {
+        char *threadgrp_string = argv[i];
+        i++;
+        if (i >= argc) {
+          ExEnv::errn() << "-threadgrp must be following by an argument"
+               << endl;
+          abort();
+        }
+        keyval_string = argv[i];
+        // move the threadgrp arguments to the end of argv
+        int j;
+        for (j=i+1; j<argc; j++) {
+          argv[j-2] = argv[j];
+        }
+        argv[j++] = threadgrp_string;
+        argv[j++] = keyval_string;
+        // decrement argc to hide the last two arguments
+        argc -= 2;
+        break;
+      }
+    }
+  }
+
+  if (!keyval_string) {
+    // find out if the environment gives the containing thread group
+    keyval_string = getenv("THREADGRP");
+    if (keyval_string) {
+      if (!strncmp("THREADGRP=", keyval_string, 11)) {
+        keyval_string = strchr(keyval_string, '=');
+      }
+      if (*keyval_string == '=') keyval_string++;
+    }
+  }
+
+  // if keyval input for a thread group was found, then
+  // create it.
+  if (keyval_string) {
+    if (keyval_string[0] == '\0') return 0;
+    Ref<ParsedKeyVal> strkv = new ParsedKeyVal();
+    strkv->parse_string(keyval_string);
+    Ref<DescribedClass> dc = strkv->describedclassvalue();
+    grp = dynamic_cast<ThreadGrp*>(dc.pointer());
+    if (dc.null()) {
+      ExEnv::errn() << "initial_threadgrp: couldn't find a ThreadGrp in "
+           << keyval_string << endl;
+      abort();
+    } else if (!grp) {
+      ExEnv::errn() << "initial_threadgrp: wanted ThreadGrp but got "
+           << dc->class_name() << endl;
+      abort();
+    }
+    // prevent an accidental delete
+    grp->reference();
+    strkv = 0;
+    dc = 0;
+    // accidental delete not a problem anymore since all smart pointers
+    // to grp are dead
+    grp->dereference();
+    return grp;
+  }
+
+  return 0;
+}
+
+ThreadGrp*
+ThreadGrp::clone(int nthread)
+{
+  ExEnv::errn() << "ThreadGrp::clone not supported for " << class_name()
+               << endl;
+  abort();
+  return 0;
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// ProcThreadLock class
+
+class ProcThreadLock : public ThreadLock {
+  public:
+    ProcThreadLock() {}
+    ~ProcThreadLock() {}
+
+    void lock() {}
+    void unlock() {}
+};
+
+/////////////////////////////////////////////////////////////////////////////
+// ProcThreadGrp members
+
+static ClassDesc ProcThreadGrp_cd(
+  typeid(ProcThreadGrp),"ProcThreadGrp",1,"public ThreadGrp",
+  0, create<ProcThreadGrp>, 0);
+
+ProcThreadGrp::ProcThreadGrp()
+  : ThreadGrp()
+{
+}
+
+ProcThreadGrp::ProcThreadGrp(const Ref<KeyVal>& keyval)
+  : ThreadGrp(keyval)
+{
+  if (nthread_ > 1) {
+    delete[] threads_;
+    nthread_ = 1;
+    threads_ = new Thread*[nthread_];
+  }
+}
+
+ProcThreadGrp::~ProcThreadGrp()
+{
+}
+
+int
+ProcThreadGrp::start_threads()
+{
+  if (threads_[0]) threads_[0]->run();
+  return 0;
+}
+
+int
+ProcThreadGrp::wait_threads()
+{
+  return 0;
+}
+
+Ref<ThreadLock>
+ProcThreadGrp::new_lock()
+{
+  return new ProcThreadLock;
+}
+
+ThreadGrp*
+ProcThreadGrp::clone(int nthread)
+{
+  return new ProcThreadGrp;
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+}
+
+// Local Variables:
+// mode: c++
+// c-file-style: "ETS"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/thread.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/thread.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/thread.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,165 @@
+//
+// thread.h
+//
+// Copyright (C) 1997 Limit Point Systems, Inc.
+//
+// Author: Edward Seidl <seidl@janed.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifndef _util_group_thread_h
+#define _util_group_thread_h
+
+#ifdef __GNUC__
+#pragma interface
+#endif
+
+#include <util/class/class.h>
+
+namespace sc {
+
+/** The ThreadLock abstract class provides mutex locks to be used in
+    conjunction with ThreadGrp's.  ThreadLock objects should be
+    locked and unlocked with ThreadLockHolder objects to provide
+    exception safety.
+*/
+class ThreadLock : public RefCount {
+  public:
+    ThreadLock();
+    virtual ~ThreadLock();
+
+    /// Obtain the lock.
+    virtual void lock() =0;
+    /// Release the lock.
+    virtual void unlock() =0;
+};
+
+
+/** Acquire a lock on creation and release it on destruction.
+    This should be used to lock and unlock ThreadLock objects
+    to provide exception safety.
+ */
+class ThreadLockHolder {
+    Ref<ThreadLock> lock_;
+    bool locked_;
+  public:
+    /// Acquires the lock.
+    ThreadLockHolder(const Ref<ThreadLock> &l): lock_(l) {
+      lock_->lock();
+      locked_ = true;
+    }
+    /// Release the lock before the DTOR is called, if it is still held.
+    void unlock() { if (locked_) { lock_->unlock(); locked_ = false; } }
+    /// Acquire the lock once more.
+    void lock() { if (!locked_) { lock_->lock(); locked_ = true; } }
+    /// Releases the lock if it is still held.
+    ~ThreadLockHolder() { unlock(); }
+};
+
+/** The Thread abstract class defines an interface which must be
+    implemented by classes wishing to be run as threads. */
+class Thread {
+  public:
+    Thread();
+    virtual ~Thread();
+
+    static void *run_Thread_run(void*thread);
+
+    /// This is called with the Thread is run from a ThreadGrp.
+    virtual void run() =0;
+};
+    
+/** The ThreadGrp abstract class provides a means to manage separate
+    threads of control. */
+class ThreadGrp: public DescribedClass {
+  protected:
+    Thread** threads_;
+    int nthread_;
+
+  public:
+    ThreadGrp();
+    ThreadGrp(const Ref<KeyVal>&);
+    ThreadGrp(const ThreadGrp&, int nthread = -1);
+    virtual ~ThreadGrp();
+
+    /** Assigns a Thread object to each thread.  If 0 is assigned to
+        a thread, then that thread will be skipped. */
+    virtual void add_thread(int threadnum, Thread* thread);
+    /** Like add_thread(threadnum, thread), but assign a priority that the
+        thread is to use.  The member is primarily for experimentation, the
+        priority argument is currently not well defined and ignored.  */
+    virtual void add_thread(int threadnum, Thread* thread, int priority);
+    /// The number of threads that will be run by start_thread.
+    int nthread() const { return nthread_; }
+
+    void delete_threads();
+
+    /** Starts the threads running.  Thread 0 will be run by the
+        thread that calls start_threads. */
+    virtual int start_threads() =0;
+    /** Wait for all the threads to complete.  This must be called
+        before start_threads is called again or the object is destroyed. */
+    virtual int wait_threads() =0;
+    /// Return a local object.
+    virtual Ref<ThreadLock> new_lock() =0;
+
+    /** Create a ThreadGrp like the current one.  If nthread is given, the
+        new ThreadGrp will attempt to support that number of threads, but
+        the actual number supported may be less.  If nthread is -1, the
+        number of threads in the current group will be used. */
+    virtual ThreadGrp* clone(int nthread = -1);
+
+    static void set_default_threadgrp(const Ref<ThreadGrp>&);
+    static ThreadGrp * get_default_threadgrp();
+    static ThreadGrp * initial_threadgrp(int &argc, char ** argv);
+};
+
+
+/** The ProcThreadGrp class privides a concrete thread group
+    appropriate for an environment where there is only one thread.
+*/
+class ProcThreadGrp: public ThreadGrp {
+  public:
+    ProcThreadGrp();
+    ProcThreadGrp(const Ref<KeyVal>&);
+    ~ProcThreadGrp();
+
+    int start_threads();
+    int wait_threads();
+    
+    Ref<ThreadLock> new_lock();
+
+    ThreadGrp* clone(int nthread = -1);
+};
+
+}
+
+extern "C" {
+    // a C linkage interface to run_Thread_run
+    void *Thread__run_Thread_run(void*thread);
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "ETS"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/thrtest.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/thrtest.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/thrtest.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,133 @@
+//
+// thrtest.cc
+// based on: messtest.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#include <util/keyval/keyval.h>
+#include <util/misc/formio.h>
+#include <util/misc/bug.h>
+#include <util/group/thread.h>
+
+using namespace std;
+using namespace sc;
+
+// Force linkages:
+//#ifndef __PIC__
+#ifdef PUMAGON
+#   include <util/group/thpuma.h>
+    static ForceLink<PumaThreadGrp> fl0;
+#endif
+# ifdef HAVE_PTHREAD
+#   include <util/group/thpthd.h>
+    static ForceLink<PthreadThreadGrp> fl2;
+# endif
+//#endif
+
+class TestThread: public Thread {
+  private:
+    Ref<ThreadLock> lock;
+  public:
+    static int count;
+    TestThread(const Ref<ThreadLock> &l): lock(l) {}
+    void run();
+    int n() const { return 1000000; }
+};
+
+int TestThread::count = 0;
+
+void
+TestThread::run()
+{
+  for (int i=0; i<n(); i++) {
+      lock->lock();
+      count++;
+      lock->unlock();
+    }
+}
+
+int
+main(int argc, char**argv)
+{
+  int i;
+
+  Ref<ThreadGrp> grp = ThreadGrp::initial_threadgrp(argc, argv);
+
+  Ref<Debugger> debugger;
+
+  if (grp.null()) {
+      const char* input = SRCDIR "/thrtest.in";
+      const char* keyword = "thread";
+
+      if (argc >= 2) input = argv[1];
+      if (argc >= 3) keyword = argv[2];
+
+      Ref<KeyVal> keyval = new ParsedKeyVal(input);
+
+      grp << keyval->describedclassvalue(keyword);
+
+      debugger << keyval->describedclassvalue(":debug");
+
+      if (grp.null()) {
+          cerr << scprintf("Couldn't initialize ThreadGrp\n");
+          abort();
+        }
+    }
+
+  if (debugger.nonnull()) {
+      debugger->set_exec(argv[0]);
+      debugger->set_prefix(0);
+    }
+
+  Debugger::set_default_debugger(debugger);
+
+  TestThread **thr = new TestThread*[grp->nthread()];
+  Ref<ThreadLock> lock = grp->new_lock();
+  for (i=0; i<grp->nthread(); i++) {
+      thr[i] = new TestThread(lock);
+      grp->add_thread(i,thr[i]);
+    }
+
+  grp->start_threads();
+  grp->wait_threads();
+
+  int right_count = 0;
+  for (i=0; i<grp->nthread(); i++) {
+      right_count += thr[i]->n();
+      delete thr[i];
+    }
+  delete[] thr;
+
+  cout << "count = " << TestThread::count << " (expected " << right_count << ")" << endl;
+
+  return 0;
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/topology.cc
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/topology.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/topology.cc	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,74 @@
+//
+// topology.cc
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUG__
+#pragma implementation
+#endif
+
+#include <util/group/message.h>
+#include <util/group/topology.h>
+
+using namespace sc;
+
+static ClassDesc GlobalMsgIter_cd(
+  typeid(GlobalMsgIter),"GlobalMsgIter",1,"public DescribedClass",
+  0, 0, 0);
+
+GlobalMsgIter::GlobalMsgIter(int nproc, int me, int root)
+{
+  nproc_ = nproc;
+  me_ = me;
+  root_ = root;
+  forwards();
+}
+
+GlobalMsgIter::~GlobalMsgIter()
+{
+}
+
+static ClassDesc MachineTopology_cd(
+  typeid(MachineTopology),"MachineTopology",1,"public DescribedClass",
+  0, 0, 0);
+
+MachineTopology::MachineTopology()
+{
+}
+
+MachineTopology::MachineTopology(const Ref<KeyVal>&)
+{
+}
+
+MachineTopology::~MachineTopology()
+{
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
Index: ThirdParty/mpqc_open/src/lib/util/group/topology.h
===================================================================
--- ThirdParty/mpqc_open/src/lib/util/group/topology.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
+++ ThirdParty/mpqc_open/src/lib/util/group/topology.h	(revision 860145ede0311b3539e419277d8ca4a3091f40b2)
@@ -0,0 +1,87 @@
+//
+// topology.h
+//
+// Copyright (C) 1996 Limit Point Systems, Inc.
+//
+// Author: Curtis Janssen <cljanss@limitpt.com>
+// Maintainer: LPS
+//
+// This file is part of the SC Toolkit.
+//
+// The SC Toolkit is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Library General Public License as published by
+// the Free Software Foundation; either version 2, or (at your option)
+// any later version.
+//
+// The SC Toolkit is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public License
+// along with the SC Toolkit; see the file COPYING.LIB.  If not, write to
+// the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+//
+// The U.S. Government is granted a limited license as per AL 91-7.
+//
+
+#ifdef __GNUG__
+#pragma interface
+#endif
+
+#ifndef _util_group_topology_h
+#define _util_group_topology_h
+
+#include <util/class/class.h>
+#include <util/keyval/keyval.h>
+
+namespace sc {
+
+class GlobalMsgIter: public DescribedClass {
+  protected:
+    int me_;
+    int nproc_;
+    int root_;
+    int i_;
+    int n_; // the number of steps--intialized by derived class CTORs
+    int fwd_;
+
+    // for sending messages in the forward direction (like a bcast)
+    virtual int fwdsendto() = 0;
+    virtual int fwdsend() = 0;
+    virtual int fwdrecvfrom() = 0;
+    virtual int fwdrecv() = 0;
+  public:
+    GlobalMsgIter(int nproc, int me, int root = 0);
+    ~GlobalMsgIter();
+    void backwards() { fwd_ = 0; i_ = n_ - 1; }
+    void forwards() { fwd_ = 1; i_ = 0; }
+    void next() { if (fwd_) i_++; else i_--; }
+    int done() { return i_<0 || i_>=n_; }
+    int n() { return n_; }
+    int sendto() { return fwd_?fwdsendto():fwdrecvfrom(); }
+    int send() { return fwd_?fwdsend():fwdrecv(); }
+    int recvfrom() { return fwd_?fwdrecvfrom():fwdsendto(); }
+    int recv() { return fwd_?fwdrecv():fwdsend(); }
+};
+
+
+class MessageGrp;
+class MachineTopology: public DescribedClass {
+  public:
+    MachineTopology();
+    MachineTopology(const Ref<KeyVal>&);
+    ~MachineTopology();
+
+    virtual Ref<GlobalMsgIter> global_msg_iter(const Ref<MessageGrp>&,
+                                             int target) = 0;
+};
+
+}
+
+#endif
+
+// Local Variables:
+// mode: c++
+// c-file-style: "CLJ"
+// End:
