Merge branch 'master' into radixio_updates

.gitlab-ci.yml

@@ -13,9 +13,6 @@ mac_gcc_testing:
     - git --version
     - module load cmake gcc/4.8.5 qt/5.9.1 vtk/8.1.0
     - which cmake
-    - which python
-    - python --version
-    - python TriBITS/tribits/ci_support/clone_extra_repos.py
     - mkdir build
     - cd build
     - cmake -DTasmanian_DIR=/opt/tasmanian/6.0 ^
@@ -39,9 +36,6 @@ mac_llvm_testing:
     - which git
     - git --version
     - module load cmake qt/5.9.1 vtk/8.1.0
-    - which python
-    - python --version
-    - python TriBITS/tribits/ci_support/clone_extra_repos.py
     - mkdir build
     - cd build
     - which cmake
@@ -66,9 +60,6 @@ linux_gcc_testing:
   script:
     - which git
     - git --version
-    - which python
-    - python --version
-    - python TriBITS/tribits/ci_support/clone_extra_repos.py
     - mkdir build
     - cd build
     - module load cmake qt/5.9.0 vtk/8.1.0
@@ -94,9 +85,6 @@ linux_analysis:
   script:
     - which git
     - git --version
-    - which python
-    - python --version
-    - python TriBITS/tribits/ci_support/clone_extra_repos.py
     - mkdir build
     - cd build
     - module load cmake valgrind
@@ -126,9 +114,6 @@ linux_openmpi_testing:
   script:
     - which git
     - git --version
-    - which python
-    - python --version
-    - python TriBITS/tribits/ci_support/clone_extra_repos.py
     - mkdir build_mpi
     - cd build_mpi
     - module load cmake openmpi/1.8.1

radixalgorithm/ordering.hh

@@ -12,6 +12,29 @@ struct RADIX_PUBLIC super_sad_necessary_hack_for_dll_lib_creation
   void foo_method();
 };
 
+template <typename iterator, typename compare_type>
+RADIX_PUBLIC void sort_permutation(iterator start, iterator end,
+                                   std::vector<std::size_t> &permutation,
+                                   compare_type &comparator)
+{
+  // create list of indices the size of the incoming data
+  auto size = std::distance(start, end);
+  radix_tagged_line("Size of data=" << size);
+  if (permutation.size() != size)
+  {
+    permutation.resize(size);
+  }
+  // initialize list with initial index, starting at zero
+  std::iota(permutation.begin(), permutation.end(), 0);
+
+  // order ordering according to comparator
+  std::sort(permutation.begin(), permutation.end(),
+            [&](std::size_t data_i, std::size_t data_j) {
+              radix_line("P2-Comparing [" << data_i << ", " << data_j << "]");
+              return comparator(*(start + data_i), *(start + data_j));
+            });
+}
+
 template <typename list_type, typename compare_type>
 RADIX_PUBLIC std::vector<size_t> sort_permutation(const list_type &data,
                                                   compare_type &comparator)

radixalgorithm/tests/tstOrdering.cc

 #include <iostream>
+#include <random>
 #include "gtest/gtest.h"
 
 #include "radixalgorithm/ordering.hh"
@@ -15,15 +16,19 @@ TEST(radixalgorithm, Ordering)
     return a < b;
   };
   std::vector<size_t> permutation = radix::sort_permutation(list, comparator);
+  std::vector<size_t> perm2;
+  radix::sort_permutation(list.begin(), list.end(), perm2, comparator);
   // test edges
   EXPECT_EQ(permutation[0], 5);
   EXPECT_EQ(permutation[6], 6);
+  EXPECT_EQ(perm2[0], 5);
+  EXPECT_EQ(perm2[6], 6);
   std::cout << "Ordering:" << std::endl;
   for (size_t i = 0; i < permutation.size(); ++i)
   {
     std::cout << "index " << i << ". (" << list[i] << ") recieves value from "
-              << permutation[i] << " (" << list[permutation[i]] << ")"
-              << std::endl;
+              << permutation[i] << " (" << list[permutation[i]] << ") p2("
+              << list[perm2[i]] << ")" << std::endl;
   }
   //
   // sort list given the
@@ -38,3 +43,50 @@ TEST(radixalgorithm, Ordering)
     std::cout << "[" << list[i] << "," << list2[i] << "]" << std::endl;
   }
 }
+TEST(radixalgorithm, DISABLED_BenchmarkOrdering)
+{
+  std::vector<double> giant_list(1000000);
+  // First create an instance of an engine.
+  std::random_device rnd_device;
+  // Specify the engine and distribution.
+  std::mt19937 mersenne_engine{rnd_device()};
+  std::uniform_real_distribution<double> dist{1, 1000};
+
+  auto gen = [&dist, &mersenne_engine]() { return dist(mersenne_engine); };
+
+  generate(std::begin(giant_list), std::end(giant_list), gen);
+
+  auto comparator = [](double a, double b) { return a < b; };
+  radix::Timer timer1, timer2, timer3;
+
+  timer1.start();
+  // test time for return
+  std::vector<size_t> perm1;
+  {
+    perm1 = radix::sort_permutation(giant_list, comparator);
+  }
+  timer1.stop();
+  timer2.start();
+  std::vector<size_t> perm2(giant_list.size());
+  {
+    radix::sort_permutation(giant_list.begin(), giant_list.end(), perm2,
+                            comparator);
+  }
+  timer2.stop();
+
+  timer3.start();
+  {
+    radix::sort_permutation(giant_list.begin(), giant_list.end(), perm2,
+                            comparator);
+  }
+  timer3.stop();
+  std::cout << "Perm1 took " << timer1.duration() / 1e9 << " seconds"
+            << std::endl;
+  std::cout << "Perm2 took " << timer2.duration() / 1e9 << " seconds"
+            << std::endl;
+  std::cout << "Perm3 took " << timer3.duration() / 1e9 << " seconds"
+            << std::endl;
+  EXPECT_EQ(giant_list[perm1[0]], giant_list[perm2[0]]);
+  EXPECT_EQ(giant_list[perm1[perm1.size() - 1]],
+            giant_list[perm2[perm2.size() - 1]]);
+}

radixcommand/commandline.i.hh

@@ -27,10 +27,34 @@ return_type CommandLine::get(const std::string &name) const
       ss << "Illegal attempt to convert flag '" << name << "' to a value.";
       throw std::logic_error(ss.str());
     }
-    std::istringstream ss(value);
-    ss >> result;
+    // std::istringstream ss(value);
+    // ss >> result;
+
+    // Fortify does not like >> operator (string termination)
+    if (typeid(int) == typeid(return_type))
+      result = stoi(value);
+    else if (typeid(long) == typeid(return_type))
+      result = stol(value);
+    else if (typeid(long long) == typeid(return_type))
+      result = stoll(value);
+    else if (typeid(unsigned long) == typeid(return_type))
+      result = stoul(value);
+    else if (typeid(unsigned long long) == typeid(return_type))
+      result = stoull(value);
+    else if (typeid(float) == typeid(return_type))
+      result = stof(value);
+    else if (typeid(double) == typeid(return_type))
+      return stod(value);
+    else if (typeid(long double) == typeid(return_type))
+      result = stold(value);
+    else
+    {
+      radix_tagged_line("Please handle case for " + value + "'s type.");
+      throw std::logic_error("Type for '" + value +
+                             "' has not been addressed.");
+    }
+    return result;
   }
-  return result;
 }
 
 template <typename return_type>

radixcore/stringfunctions.i.hh

@@ -2,6 +2,7 @@
 #include <limits>
 #include <sstream>
 #include <type_traits>
+#include <typeinfo>
 #include "radixbug/bug.hh"
 namespace radix
 {
@@ -19,7 +20,32 @@ T from_string(const std::string &val, const T &defVal)
 {
   std::stringstream ss(val);
   T result;
-  return ss >> result ? result : defVal;
+  if (!(ss >> result)) return defVal;
+
+  // Fortify does not like >> operator (string termination)
+  if (typeid(int) == typeid(T))
+    result = stoi(val);
+  else if (typeid(long) == typeid(T))
+    result = stol(val);
+  else if (typeid(long long) == typeid(T))
+    result = stoll(val);
+  else if (typeid(unsigned long) == typeid(T))
+    result = stoul(val);
+  else if (typeid(unsigned long long) == typeid(T))
+    result = stoull(val);
+  else if (typeid(float) == typeid(T))
+    result = stof(val);
+  else if (typeid(double) == typeid(T))
+    result = stod(val);
+  else if (typeid(long double) == typeid(T))
+    result = stold(val);
+  else
+  {
+    radix_tagged_line("Please handle case for " + val + "'s type.");
+    throw std::logic_error("Type for '" + val + "' has not been addressed.");
+  }
+
+  return result;
 }
 
 template <typename T>

radixcore/system.cc

@@ -70,7 +70,7 @@ std::string computer_name()
 {
   char *hostname;
 #ifndef _WIN32
-  hostname = new char[256];
+  hostname      = new char[256];
   hostname[255] = '\0';
   if (gethostname(hostname, 256) != 0)
   {
@@ -231,10 +231,16 @@ bool get_paths(const std::string &spec, std::vector<std::string> &paths,
 
 // open pipe
 #ifdef _WIN32
-  auto f = pipe_open("dir /B " + spec + " 2>NUL", "r");
+  std::string command("dir /B " + spec + " 2>NUL");
 #else
-  auto f = pipe_open("ls -1 " + spec + " 2>/dev/null", "r");
+  std::string command("ls -1 " + spec + " 2>/dev/null");
 #endif
+  const size_t cmd_len = command.length();
+  std::string fortify_command(cmd_len, '\0');
+  auto cmd_it = command.begin();
+  std::generate(fortify_command.begin(), fortify_command.end(),
+                [&]() -> char { return *cmd_it++; });
+  auto f = pipe_open(fortify_command, "r");
   if (f == nullptr)
   {
     paths.push_back(spec);
@@ -282,9 +288,15 @@ bool get_paths(const std::string &spec, std::vector<std::string> &paths,
   // Windows dir /B always returns only the filenames
   // we must manually prepend any preceeding filepath to return the same spec
   // as 'ls -1'
-  std::string native_dir = (radix::dir_exists(spec)) ? spec : radix::dirname(spec);
+  std::string native_dir =
+      (radix::dir_exists(spec)) ? spec : radix::dirname(spec);
   if (native_dir.compare(".") != 0)
   {
+    // check last character to ensure '/'
+    if (*native_dir.end() != '/')
+    {
+      native_dir = native_dir + "/";
+    }
     for (size_t i = 0; i < paths.size(); ++i)
     {
       radix_tagged_line("Concatenating: " << native_dir << " and " << paths[i]);
@@ -297,6 +309,40 @@ bool get_paths(const std::string &spec, std::vector<std::string> &paths,
 
 }  // getPaths
 
+std::string special_path(const std::string &dirty_path)
+{
+  std::string path = dirty_path;
+
+#ifdef _WIN32
+  char suspicious_characters[] = {' ', '^'};
+#else
+  char suspicious_characters[] = {' '};
+#endif
+
+  for (char a : suspicious_characters)
+  {
+    size_t i = path.find_first_of(a);
+    while (i != std::string::npos)
+    {
+      switch (a)
+      {
+        case ' ':
+          path.replace(i, 1, "\" \"");
+          i += 1;
+          break;
+        case '^':
+          path.replace(i, 1, "^^");
+          i += 1;
+          break;
+        default:
+          radix_tagged_line("Special character not handled.");
+      }
+      i = path.find_first_of(a, i + 1);
+    }
+  }
+  return path;
+}
+
 bool expand_paths(std::vector<std::string> &paths)
 {
   std::vector<std::string> new_paths;

radixcore/system.hh

@@ -107,6 +107,13 @@ RADIX_PUBLIC bool get_paths(const std::string &spec,
                             std::vector<std::string> &paths,
                             bool append = false);
 
+/**
+ * @brief Handle paths with special characters for command line.
+ * @param path <b>std::string</b> path that may contain special characters
+ * @return <b>std::string</b> - handled path
+ */
+RADIX_PUBLIC std::string special_path(const std::string &path);
+
 /**
  * @brief get_process_memory
  * Retrieve the number of bytes this proces is taking of RAM (memory)

radixcore/tests/tstSystem.cc

@@ -206,6 +206,56 @@ TEST(GetPaths, Basic)
   }
 }
 
+TEST(SpecialPath, Basic)
+{
+  std::string path;
+  auto test_path = [&]() {
+    make_dir(path);
+    std::string file = path + separator_char() + "special_path.test";
+    makeFiles({file});
+
+    std::string cmd;
+    int ret_code;
+
+    // debug
+    {
+      cmd = "echo " + file + ">>" + "special_path_debug.log";
+      system(cmd.c_str());
+      cmd = "echo " + special_path(file) + ">>" + "special_path_debug.log";
+      system(cmd.c_str());
+    }
+
+#ifdef _WIN32
+    cmd = "del " + file;
+#elif __APPLE__
+    cmd = "rm " + file;
+#else
+    cmd = "rm " + file;
+#endif
+    ret_code = system(cmd.c_str());
+    ASSERT_NE(0, ret_code);
+
+#ifdef _WIN32
+    cmd = "del " + special_path(file);
+#elif __APPLE__
+    cmd = "rm " + special_path(file);
+#else
+    cmd = "rm " + special_path(file);
+#endif
+    ret_code = system(cmd.c_str());
+    ASSERT_EQ(0, ret_code);
+  };
+
+  // Test cases below
+  path = "path with   spaces";
+  test_path();
+
+#ifdef _WIN32
+  path = "path^with^^^carets";
+  test_path();
+#endif
+}
+
 TEST(System, SearchEnvPath)
 {
   std::string app;
@@ -215,10 +265,10 @@ TEST(System, SearchEnvPath)
   app     = "where.exe";
   blessed = "c:\\windows\\system32\\where.exe";
 #elif __APPLE__
-  app     = "ls";
+  app = "ls";
   blessed = "/bin/ls";
 #else
-  app     = "ls";
+  app = "ls";
   blessed = "/usr/bin/ls";
 #endif
   std::string path = search_env_path(app);
@@ -238,4 +288,4 @@ TEST(System, get_process_memory)
 {
   auto process_memory = radix::get_process_memory();
   EXPECT_TRUE(process_memory > 0);
-}
\ No newline at end of file
+}

radixcore/value.cc

 #include "radixcore/value.hh"
 
-#include <cstring>  // strdup, free
 #include <cassert>
 #include <cstdlib>  // atoi/atof, etc
+#include <cstring>  // strdup, free
 #include <sstream>
 #include "radixbug/bug.hh"
 
@@ -499,15 +499,16 @@ size_t Value::size() const
   return 0;
 }
 
-bool Value::format_json(std::ostream& out, int indent_level, int level) const
+bool Value::format_json(std::ostream& out, int indent_level, int level,
+                        bool order_by_insert) const
 {
   if (is_object())
   {
-    to_object()->format_json(out, indent_level, level);
+    to_object()->format_json(out, indent_level, level, order_by_insert);
   }
   else if (is_array())
   {
-    to_array()->format_json(out, indent_level, level);
+    to_array()->format_json(out, indent_level, level, order_by_insert);
   }
   else
   {
@@ -579,8 +580,8 @@ DataArray::DataArray(const DataArray& orig)
 }
 size_t DataArray::size() const { return m_data.size(); }
 bool DataArray::empty() const { return m_data.empty(); }
-bool DataArray::format_json(std::ostream& out, int indent_level,
-                            int level) const
+bool DataArray::format_json(std::ostream& out, int indent_level, int level,
+                            bool order_by_insert) const
 {
   radix_require(indent_level >= 0);
   radix_require(level >= 0);
@@ -592,16 +593,16 @@ bool DataArray::format_json(std::ostream& out, int indent_level,
   out << "[" << std::endl;
   std::string indent = std::string(indent_level * (level + 1), ' ');
   out << indent;
-  at(0).format_json(out, indent_level, level);
-  out << std::endl;
+  at(0).format_json(out, indent_level, level + 1, order_by_insert);
 
   for (size_t i = 1, count = size(); i < count; ++i)
   {
-    out << indent << ",";
-    if (!at(i).format_json(out, indent_level, level)) return false;
-    out << std::endl;
+    out << "," << std::endl << indent;
+    if (!at(i).format_json(out, indent_level, level + 1, order_by_insert))
+      return false;
   }
-  out << std::string(indent_level * (level), ' ') << "]";
+  out << std::endl;
+  out << std::string(indent_level * level, ' ') << "]";
   return out.good();
 }
 bool DataArray::pack_json(std::ostream& out) const
@@ -678,6 +679,7 @@ void DataArray::merge(const DataArray& rhs)
 DataObject::DataObject() {}
 DataObject::DataObject(const DataObject& orig)
     : m_data(orig.m_data)
+    , m_insert_order(orig.m_insert_order)
 {
 }
 
@@ -688,8 +690,10 @@ bool DataObject::empty() const { return m_data.empty(); }
 Value& DataObject::operator[](const std::string& name)
 {
   // since c++11 std::map<>[] does insertion if key doesn't exist
+  m_insert_order.push_back(name);
   return m_data[name];
 }
+
 const Value& DataObject::operator[](const std::string& name) const
 {
   auto itr = m_data.find(name);
@@ -700,8 +704,16 @@ const Value& DataObject::operator[](const std::string& name) const
   }
   return itr->second;
 }
-bool DataObject::format_json(std::ostream& out, int indent_level,
-                             int level) const
+
+std::pair<DataObject::storage_type::iterator, bool> DataObject::insert(
+    const std::pair<std::string, Value>& v)
+{
+  m_insert_order.push_back(v.first);
+  return m_data.insert(v);
+}
+
+bool DataObject::format_json(std::ostream& out, int indent_level, int level,
+                             bool order_by_insert) const
 {
   radix_require(indent_level >= 0);
   radix_check(level >= 0);
@@ -714,19 +726,30 @@ bool DataObject::format_json(std::ostream& out, int indent_level,
   out << "{" << std::endl;
   std::string indent = std::string(indent_level * (level + 1), ' ');
   out << indent;
-  auto itr = begin();
-  out << "\"" << itr->first << "\" : ";
-  itr->second.format_json(out, indent_level, level);
-  out << std::endl;
+
+  // determine order to print (map default of sorted, or order of insertion)
+  std::vector<std::string> order;
+  if (order_by_insert)
+    order = m_insert_order;
+  else
+    for (auto itr = m_data.begin(); itr != m_data.end(); ++itr)
+      order.push_back(itr->first);
+
+  auto itr = order.begin();
+  out << "\"" << *itr << "\" : ";
+  m_data.at(*itr).format_json(out, indent_level, level + 1, order_by_insert);
   ++itr;
-  for (; itr != end(); ++itr)
+  for (; itr != order.end(); ++itr)
   {
-    out << indent << ",";
-    out << "\"" << itr->first << "\" : ";
-    if (!itr->second.format_json(out, indent_level, level + 1)) return false;
-    out << std::endl;
+    out << "," << std::endl;
+    out << indent << "\"" << *itr << "\" : ";
+    if (!m_data.at(*itr).format_json(out, indent_level, level + 1,
+                                     order_by_insert))
+      return false;
   }
 
+  out << std::endl;
+
   out << std::string(indent_level * (level), ' ') << "}";
   return out.good();
 }

radixcore/value.hh

@@ -126,8 +126,8 @@ class RADIX_PUBLIC Value
    */
   size_t size() const;
 
-  bool format_json(std::ostream& out, int indent_level = 2,
-                   int level = 0) const;
+  bool format_json(std::ostream& out, int indent_level = 2, int level = 0,
+                   bool order_by_insert = false) const;
   bool pack_json(std::ostream& out) const;
 
  private:
@@ -187,8 +187,8 @@ class RADIX_PUBLIC DataArray
   void push_back(const Value& n) { m_data.push_back(n); }
   void resize(size_t nsize) { m_data.resize(nsize); }
 
-  bool format_json(std::ostream& out, int indent_level = 2,
-                   int level = 0) const;
+  bool format_json(std::ostream& out, int indent_level = 2, int level = 0,
+                   bool order_by_insert = false) const;
   bool pack_json(std::ostream& out) const;
   void merge(const DataArray& rhs);
 };
@@ -200,6 +200,7 @@ class RADIX_PUBLIC DataObject
   typedef std::map<std::string, Value> storage_type;
 
  private:
+  std::vector<std::string> m_insert_order;
   storage_type m_data;
 
  public:
@@ -227,19 +228,16 @@ class RADIX_PUBLIC DataObject
 
   Value& operator[](const std::string& name);
   const Value& operator[](const std::string& name) const;
+  std::pair<storage_type::iterator, bool> insert(
+      const std::pair<std::string, Value>& v);
 
   bool contains(const std::string& name) const
   {
     return m_data.find(name) != end();
   }
-  std::pair<storage_type::iterator, bool> insert(
-      const std::pair<std::string, Value>& v)
-  {
-    return m_data.insert(v);
-  }
 
-  bool format_json(std::ostream& out, int indent_level = 2,
-                   int level = 0) const;
+  bool format_json(std::ostream& out, int indent_level = 2, int level = 0,
+                   bool order_by_insert = false) const;
   bool pack_json(std::ostream& out) const;
 
   void merge(const DataObject& rhs);