Memory.cpp 19.6 KB
Newer Older
1
2
3
#include "MantidKernel/Memory.h"
#include "MantidKernel/Logger.h"

4
#include <sstream>
5

6
#ifdef __linux__
7
8
9
#include <unistd.h>
#include <fstream>
#include <malloc.h>
10
11
#include <stdio.h>
#include <sys/resource.h>
12
13
#endif
#ifdef __APPLE__
14
15
#include <malloc/malloc.h>
#include <sys/sysctl.h>
16
#include <mach/mach.h>
17
18
#include <mach/mach_host.h>
#include <mach/task.h>
19
#endif
20
#ifdef _WIN32
21
22
#include <windows.h>
#include <Psapi.h>
23
#endif
24

25
26
27
using std::size_t;
using std::string;

28
29
30
31
32
namespace Mantid {
namespace Kernel {
namespace {
/// static logger object
Logger g_log("Memory");
33
34
}

35
/// Utility function to convert memory in kiB into easy to read units.
36
template <typename TYPE> string memToString(const TYPE mem_in_kiB) {
37
38
  std::stringstream buffer;
  if (mem_in_kiB < static_cast<TYPE>(1024))
39
    buffer << mem_in_kiB << " kB";
40
  else if (mem_in_kiB < static_cast<TYPE>(100 * 1024 * 1024))
41
    buffer << (mem_in_kiB / static_cast<TYPE>(1024)) << " MB";
42
  else
43
    buffer << (mem_in_kiB / static_cast<TYPE>(1024 * 1024)) << " GB";
44
45
46
  return buffer.str();
}

47
48
// -------------------- functions for getting the memory associated with the
// process
49
/** Attempts to read the system-dependent data for a process' virtual memory
50
51
 * size and resident set size, and return the results in KB. On failure, returns
 * 0.0, 0.0
52
53
54
 * @param vm_usage :: The virtual memory usage is stored in this variable in KiB
 * @param resident_set:: The memory associated with the current process in KiB
 */
55
void process_mem_usage(size_t &vm_usage, size_t &resident_set) {
56
57
58
  vm_usage = 0;
  resident_set = 0;

59
#ifdef __linux__
60
61
  // Adapted from
  // http://stackoverflow.com/questions/669438/how-to-get-memory-usage-at-run-time-in-c
62
63
64
65
  using std::ios_base;
  using std::ifstream;

  // 'file' stat seems to give the most reliable results
66
  ifstream stat_stream("/proc/self/stat", ios_base::in);
67
68
69
70
71
72
73
74
75

  // dummy vars for leading entries in stat that we don't care about
  string pid, comm, state, ppid, pgrp, session, tty_nr;
  string tpgid, flags, minflt, cminflt, majflt, cmajflt;
  string utime, stime, cutime, cstime, priority, nice;
  string O, itrealvalue, starttime;

  // the two fields we want
  unsigned long vsize; // according to man this is %lu
76
  long rss;            // according to man this is %ld
77

78
79
80
81
  stat_stream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr >>
      tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt >> utime >>
      stime >> cutime >> cstime >> priority >> nice >> O >> itrealvalue >>
      starttime >> vsize >> rss; // don't care about the rest
82

83
84
85
  long page_size_kb = sysconf(_SC_PAGE_SIZE) /
                      1024; // in case x86-64 is configured to use 2MB pages
  vm_usage = static_cast<size_t>(vsize / static_cast<long double>(1024.0));
86
  resident_set = static_cast<size_t>(rss * page_size_kb);
87
#elif __APPLE__
88
89
90
  // Adapted from http://blog.kuriositaet.de/?p=257. No official apple docs
  // could be found
  // task_t task = MACH_PORT_NULL;
91
92
93
  struct task_basic_info t_info;
  mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT;

94
  if (KERN_SUCCESS != task_info(mach_task_self(), TASK_BASIC_INFO,
Hahn, Steven's avatar
Hahn, Steven committed
95
96
                                reinterpret_cast<task_info_t>(&t_info),
                                &t_info_count)) {
97
    return;
98
99
100
  }
  // Need to find out the system page size for next part
  vm_size_t pageSize;
101
  mach_port_t port = mach_host_self();
102
  host_page_size(port, &pageSize);
103
104
  resident_set = static_cast<size_t>(t_info.resident_size * pageSize);
  vm_usage = static_cast<size_t>(t_info.virtual_size * pageSize / 1024.0);
105
#elif _WIN32
106
107
  // Adapted from
  // http://msdn.microsoft.com/en-us/library/windows/desktop/ms682050%28v=vs.85%29.aspx
108
  DWORD pid = GetCurrentProcessId();
109
110
111
112
  HANDLE hProcess =
      OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid);
  if (NULL == hProcess)
    return;
113
  PROCESS_MEMORY_COUNTERS pmc;
114
  if (GetProcessMemoryInfo(hProcess, &pmc, sizeof(pmc))) {
115
116
117
    vm_usage = pmc.PagefileUsage / 1024;
    resident_set = pmc.WorkingSetSize / 1024;
  }
118
  CloseHandle(hProcess);
119
120
121
#endif
}

122
123
// ----------------------- functions associated with getting the memory of the
// system
124
125

#ifdef __linux__
126
127
/**
 * This function reads /proc/meminfo to get the system information.
128
129
130
131
 * @param sys_avail :: An output variable containing the available system memory
 * in KiB
 * @param sys_total :: An output variable containing the total system memory in
 * KiB
132
 */
133
bool read_mem_info(size_t &sys_avail, size_t &sys_total) {
134
  std::ifstream file("/proc/meminfo");
135
  string line;
136
  int values_found(0);
137
  // Need to set this to zero
138
  sys_avail = 0;
139
  while (getline(file, line)) {
140
    std::istringstream is(line);
141
    string tag;
142
143
144
145
    long value(0);
    is >> tag >> value;
    if (!is)
      return false;
146
    if (tag == "MemTotal:") {
147
148
      ++values_found;
      sys_total = value;
149
    } else if (tag == "MemFree:") {
150
151
      ++values_found;
      sys_avail += value;
152
    } else if (tag == "Cached:") {
153
      ++values_found;
154
      sys_avail += value;
155
156
157
158
    } else if (tag == "Buffers:") {
      ++values_found;
      sys_avail += value;
    } else
159
      continue;
160
    if (values_found == 4) {
161
162
163
164
165
166
167
168
169
      file.close();
      return true;
    }
  }
  file.close();
  return false;
}
#endif

170
#ifdef _WIN32
171
namespace { // Anonymous namespace
172

173
174
MEMORYSTATUSEX
memStatus; ///< A Windows structure holding information about memory usage
175
176
177
}
#endif

178
/** Attempts to read the system memory statistics.
179
180
181
182
 * @param sys_avail :: An output variable containing the reported available
 * system memory in this variable in KiB
 * @param sys_total :: An output variable containing the reported total system
 * memory in the system in KiB
183
 */
184
void MemoryStats::process_mem_system(size_t &sys_avail, size_t &sys_total) {
185
186
187
188
189
190
  sys_avail = 0;
  sys_total = 0;
#ifdef __linux__
  /*
   * Taken from API/MemoryManager.cpp_LINUX
   *
191
192
193
194
195
   * As usual things are more complex on Linux. I think we need to
   * take into account the value of Cached as well since, especially
   * if the system has been running for a long time, MemFree will seem
   * a lot smaller than it should be. To be completely correct we also
   * need to add the value of the "Buffers" as well.
196
   *
197
198
199
200
201
202
   * The only way I can see as to get acces to the Cached value is
   * from the /proc/meminfo file so if this is not successful I'll
   * fall back to using the sysconf method and forget the cache
   * RJT(18/2/10) : Should we be using sysinfo() here?
   */
  if (!read_mem_info(sys_avail, sys_total)) {
203
204
205
    long int totPages = sysconf(_SC_PHYS_PAGES);
    long int avPages = sysconf(_SC_AVPHYS_PAGES);
    long int pageSize = sysconf(_SC_PAGESIZE);
206
207
208
209
210
211
212
213
214
    if (totPages < 0)
      totPages = 0;
    if (avPages < 0)
      totPages = 0;
    if (pageSize < 1)
      pageSize = 1;
    // Commented out the next line as the value was being written by the one
    // after
    // sys_avail = avPages / 1024 * pageSize;
215
216
    sys_avail = totPages / 1024 * pageSize;
  }
217
218
219
220
221
222
223
224
225
  // Can get the info on the memory that we've already obtained but aren't using
  // right now
  int unusedReserved = mallinfo().fordblks / 1024;
  // unusedReserved can sometimes be negative, which wen added to a low
  // sys_avail will overflow the unsigned int.
  if (unusedReserved < 0)
    unusedReserved = 0;
  // g_log.debug() << "Linux - Adding reserved but unused memory of " <<
  // unusedReserved << " KB\n";
226
  sys_avail += unusedReserved;
227

228
#elif __APPLE__
229
230
231
232
  // Get the total RAM of the system
  uint64_t totalmem;
  size_t len = sizeof(totalmem);
  // Gives system memory in bytes
233
  int err = sysctlbyname("hw.memsize", &totalmem, &len, nullptr, 0);
234
235
  if (err)
    g_log.warning("Unable to obtain memory of system");
236
237
238
239
240
241
242
243
244
245
246
  sys_total = totalmem / 1024;

  mach_port_t port = mach_host_self();
  // Need to find out the system page size for next part
  vm_size_t pageSize;
  host_page_size(port, &pageSize);

  // Now get the amount of free memory (=free+inactive memory)
  vm_statistics vmStats;
  mach_msg_type_number_t count;
  count = sizeof(vm_statistics) / sizeof(natural_t);
Hahn, Steven's avatar
Hahn, Steven committed
247
248
  err = host_statistics(port, HOST_VM_INFO,
                        reinterpret_cast<host_info_t>(&vmStats), &count);
249
250
251
  if (err)
    g_log.warning("Unable to obtain memory statistics for this Mac.");
  sys_avail = pageSize * (vmStats.free_count + vmStats.inactive_count) / 1024;
252
253
254

  // Now add in reserved but unused memory as reported by malloc
  const size_t unusedReserved = mstats().bytes_free / 1024;
255
256
  g_log.debug() << "Mac - Adding reserved but unused memory of "
                << unusedReserved << " KB\n";
257
  sys_avail += unusedReserved;
258
#elif _WIN32
259
260
261
262
263
264
  GlobalMemoryStatusEx(&memStatus);
  if (memStatus.ullTotalPhys < memStatus.ullTotalVirtual) {
    sys_avail = static_cast<size_t>(memStatus.ullAvailPhys / 1024);
    sys_total = static_cast<size_t>(memStatus.ullTotalPhys / 1024);
  } else // All virtual memory will be physical, but a process cannot have more
         // than TotalVirtual.
265
  {
266
267
    sys_avail = static_cast<size_t>(memStatus.ullAvailVirtual / 1024);
    sys_total = static_cast<size_t>(memStatus.ullTotalVirtual / 1024);
268
  }
269
#endif
270

271
272
  g_log.debug() << "Memory: " << sys_avail << " (free), " << sys_total
                << " (total).\n";
273
274
}

275
276
/**
 * Initialize platform-dependent options for memory management.
277
278
279
280
 * On Windows this enables the low-fragmentation heap described here:
 * http://msdn.microsoft.com/en-us/library/aa366750%28v=vs.85%29.aspx
 * On Linux this enables the mmap option for malloc calls to try and release
 * memory more frequently.
281
282
 * Note that this function can only be called once
 */
283
void MemoryOptions::initAllocatorOptions() {
284
  static bool initialized(false);
285
286
  if (initialized)
    return;
287
#ifdef __linux__
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
  /* The line below tells malloc to use a different memory allocation system
  * call (mmap) to the 'usual'
  * one (sbrk) for requests above the threshold of the second argument (in
  * bytes). The effect of this
  * is that, for the current threshold value of 8*4096, storage for workspaces
  * having 4096 or greater
  * bins per spectrum will be allocated using mmap.
  * This should have the effect that memory is returned to the kernel as soon as
  * a workspace is deleted,
  * preventing things going to managed workspaces when they shouldn't. This will
  * also hopefully reduce
  * memory fragmentation.
  * Potential downsides to look out for are whether this memory allocation
  * technique makes things
  * noticeably slower and whether it wastes memory (mmap allocates in blocks of
  * the system page size.
  */
  mallopt(M_MMAP_THRESHOLD, 8 * 4096);
306
#elif _WIN32
307
  Logger memOptLogger("MemoryOptions");
Peterson, Peter's avatar
Peterson, Peter committed
308
  // Try to enable the Low Fragmentation Heap for all heaps
309
310
  // Bit of a brute force approach, but don't know which heap workspace data
  // ends up on
Peterson, Peter's avatar
Peterson, Peter committed
311
312
313
  HANDLE hHeaps[1025];
  // Get the number of heaps
  const DWORD numHeap = GetProcessHeaps(1024, hHeaps);
314
315
316
317
318
319
  memOptLogger.debug() << "Number of heaps: " << numHeap
                       << "\n"; // GetProcessHeaps(0, NULL) << "\n";
  ULONG ulEnableLFH = 2;        // 2 = Low Fragmentation Heap
  for (DWORD i = 0; i < numHeap; i++) {
    if (!HeapSetInformation(hHeaps[i], HeapCompatibilityInformation,
                            &ulEnableLFH, sizeof(ulEnableLFH))) {
320
      memOptLogger.debug() << "Failed to enable the LFH for heap " << i << "\n";
Peterson, Peter's avatar
Peterson, Peter committed
321
    }
322
323
  }
#endif
324
325
326
327
328
329
330
331
332
  initialized = true;
}

// ------------------ The actual class ----------------------------------------

/**
 * Constructor
 * @param ignore :: Which memory stats should be ignored.
 */
333
334
MemoryStats::MemoryStats(const MemoryStatsIgnore ignore)
    : vm_usage(0), res_usage(0), total_memory(0), avail_memory(0) {
335
336
337
338

#ifdef _WIN32
  memStatus.dwLength = sizeof(MEMORYSTATUSEX);
#endif
339

340
341
342
343
  this->ignoreFields(ignore);
  this->update();
}

344
/** Update the structure with current information, taking into account what is
345
 * to be ignored.
346
347
 * This call is thread-safe (protected by a mutex).
 * Note: This takes about 0.1 ms on a Ubuntu 10.10 system.
348
 */
349
void MemoryStats::update() {
350
  std::lock_guard<std::mutex> lock(MemoryStats::mutexMemory);
351
  // get what is used by the process
352
  if (this->ignore != MEMORY_STATS_IGNORE_PROCESS) {
353
    process_mem_usage(this->vm_usage, this->res_usage);
354
  }
355
356

  // get the system information
357
  if (this->ignore != MEMORY_STATS_IGNORE_SYSTEM) {
358
359
    process_mem_system(this->avail_memory, this->total_memory);
  }
360
361
}

362
363
364
365
/**
 * Set the fields to ignore
 * @param ignore :: An enumeration giving the fields to ignore
 */
366
void MemoryStats::ignoreFields(const MemoryStatsIgnore ignore) {
367
368
369
  this->ignore = ignore;
}

370
371
372
373
/**
 * Returns the virtual memory usage as a string
 * @returns A string containing the amount of virtual memory usage
 */
374
string MemoryStats::vmUsageStr() const { return memToString(this->vm_usage); }
375

376
377
378
379
/**
 * Returns the resident memory used by the current process
 * @returns A string containing the amount of memory the process is using
 */
380
string MemoryStats::resUsageStr() const { return memToString(this->res_usage); }
381

382
383
384
385
/**
 * Returns the total memory of the system as a string
 * @returns A string containing the total amount of memory on the system
 */
386
string MemoryStats::totalMemStr() const {
387
388
389
  return memToString(this->total_memory);
}

390
391
392
393
/**
 * Returns the available memory of the system as a string
 * @returns A string containing the amount of available memory on the system
 */
394
string MemoryStats::availMemStr() const {
395
396
397
  return memToString(this->avail_memory);
}

398
/**
399
 * Returns the total memory of the system
400
401
 * @returns An unsigned containing the total amount of memory on the system in
 * kiB
402
 */
403
size_t MemoryStats::totalMem() const { return this->total_memory; }
404

405
/**
406
 * Returns the available memory of the system in kiB
407
408
 * @returns An unsigned containing the available amount of memory on the system
 * in kiB
409
 */
410
size_t MemoryStats::availMem() const { return this->avail_memory; }
411

412
413
414
415
/**
 * Returns the memory usage of the current process in kiB
 * @returns An unsigned containing the memory used by the current process in kiB
 */
416
size_t MemoryStats::residentMem() const { return this->res_usage; }
417
418
419

/**
 * Returns the virtual memory usage of the current process in kiB
420
421
 * @returns An unsigned containing the virtual memory used by the current
 * process in kiB
422
423
 */

424
size_t MemoryStats::virtualMem() const { return this->vm_usage; }
425

426
/**
427
428
 * Returns the reserved memory that has not been factored into the available
 * memory
429
430
431
 * calculation.
 * NOTE: On Windows this can be a lengthy calculation as it involves
 * adding up the reserved space DWORD length at a time. Call only when necessary
432
433
 * On other systems this will return 0 as it has already been factored in to the
 * available
434
435
436
 * memory calculation
 * @returns An extra area of memory that can still be allocated.
 */
437
size_t MemoryStats::reservedMem() const {
438
#ifdef _WIN32
Peterson, Peter's avatar
Peterson, Peter committed
439
440
441
442
  MEMORY_BASIC_INFORMATION info; // Windows structure
  char *addr = NULL;
  size_t unusedReserved = 0; // total reserved space
  DWORDLONG size = 0;
443
444
445
  GlobalMemoryStatusEx(&memStatus);
  DWORDLONG GB2 =
      memStatus.ullTotalVirtual; // Maximum memory available to the process
Peterson, Peter's avatar
Peterson, Peter committed
446
447

  // Loop over all virtual memory to find out the status of every block.
448
449
  do {
    VirtualQuery(addr, &info, sizeof(MEMORY_BASIC_INFORMATION));
Peterson, Peter's avatar
Peterson, Peter committed
450
451

    // Count up the total size of reserved but unused blocks
452
453
    if (info.State == MEM_RESERVE)
      unusedReserved += info.RegionSize;
Peterson, Peter's avatar
Peterson, Peter committed
454

455
456
    addr +=
        info.RegionSize; // Move up to the starting address for the next call
Peterson, Peter's avatar
Peterson, Peter committed
457
    size += info.RegionSize;
458
  } while (size < GB2);
Peterson, Peter's avatar
Peterson, Peter committed
459
460
461
462

  // Convert from bytes to KB
  unusedReserved /= 1024;

463
464
465
466
467
468
469
470
471
472
  return unusedReserved;
#else
  return 0;
#endif
}

/**
 * The ratio of available to total system memory as a number between 0-100.
 * @returns A percentage
 */
473
474
475
double MemoryStats::getFreeRatio() const {
  return 100. * static_cast<double>(this->avail_memory) /
         static_cast<double>(this->total_memory);
476
477
478
}

/// Convenience function for writting out to stream.
479
480
std::ostream &operator<<(std::ostream &out, const MemoryStats &stats) {
  if (stats.ignore != MEMORY_STATS_IGNORE_PROCESS) {
481
482
483
484
485
486
487
488
    out << "virtual[" << stats.vmUsageStr() << "] ";
    out << "resident[" << stats.resUsageStr() << "] ";
  }
  if (stats.ignore != MEMORY_STATS_IGNORE_SYSTEM) {
    out << "available[" << stats.availMemStr() << "] ";
    out << "total[" << stats.totalMemStr() << "] ";
  }
  return out;
489
490
}

491
492
493
494
495
496
497
498
/**
 * @returns the peak (maximum so far) resident set size (physical
 * memory use) measured in bytes, or zero if the value cannot be
 * determined on this OS.
 *
 * This was adopted from
 *http://nadeausoftware.com/articles/2012/07/c_c_tip_how_get_process_resident_set_size_physical_memory_use
 */
499
size_t MemoryStats::getPeakRSS() const {
500
501
502
503
504
505
#if defined(_WIN32)
  /* Windows -------------------------------------------------- */
  PROCESS_MEMORY_COUNTERS info;
  GetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info));
  return (size_t)info.PeakWorkingSetSize;

Tom Perkins's avatar
Tom Perkins committed
506
#elif(defined(_AIX) || defined(__TOS__AIX__)) ||                               \
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
    (defined(__sun__) || defined(__sun) ||                                     \
     defined(sun) && (defined(__SVR4) || defined(__svr4__)))
  /* AIX and Solaris ------------------------------------------ */
  struct psinfo psinfo;
  int fd = -1;
  if ((fd = open("/proc/self/psinfo", O_RDONLY)) == -1)
    return (size_t)0L; /* Can't open? */
  if (read(fd, &psinfo, sizeof(psinfo)) != sizeof(psinfo)) {
    close(fd);
    return (size_t)0L; /* Can't read? */
  }
  close(fd);
  return (size_t)(psinfo.pr_rssize * 1024L);

#elif defined(__unix__) || defined(__unix) || defined(unix) ||                 \
    (defined(__APPLE__) && defined(__MACH__))
  /* BSD, Linux, and OSX -------------------------------------- */
  struct rusage rusage;
  getrusage(RUSAGE_SELF, &rusage);
#if defined(__APPLE__) && defined(__MACH__)
527
  return static_cast<size_t>(rusage.ru_maxrss);
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
#else
  return (size_t)(rusage.ru_maxrss * 1024L);
#endif

#else
  /* Unknown OS ----------------------------------------------- */
  return (size_t)0L; /* Unsupported. */
#endif
}

/**
 * @returns the current resident set size (physical memory use) measured
 * in bytes, or zero if the value cannot be determined on this OS.
 *
 * This was adopted from
 *http://nadeausoftware.com/articles/2012/07/c_c_tip_how_get_process_resident_set_size_physical_memory_use
 */
545
size_t MemoryStats::getCurrentRSS() const {
546
547
548
549
550
551
552
553
554
555
#if defined(_WIN32)
  /* Windows -------------------------------------------------- */
  PROCESS_MEMORY_COUNTERS info;
  GetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info));
  return (size_t)info.WorkingSetSize;

#elif defined(__APPLE__) && defined(__MACH__)
  /* OSX ------------------------------------------------------ */
  struct mach_task_basic_info info;
  mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;
Hahn, Steven's avatar
Hahn, Steven committed
556
557
  if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO,
                reinterpret_cast<task_info_t>(&info),
558
                &infoCount) != KERN_SUCCESS)
559
560
    return static_cast<size_t>(0L); /* Can't access? */
  return static_cast<size_t>(info.resident_size);
561
562
563
564
565
566
567
568

#elif defined(__linux__) || defined(__linux) || defined(linux) ||              \
    defined(__gnu_linux__)
  /* Linux ---------------------------------------------------- */
  long rss = 0L;
  FILE *fp = NULL;
  if ((fp = fopen("/proc/self/statm", "r")) == NULL)
    return (size_t)0L; /* Can't open? */
569
  if (fscanf(fp, "%*s%20ld", &rss) != 1) {
570
571
572
573
574
575
576
577
578
579
580
581
    fclose(fp);
    return (size_t)0L; /* Can't read? */
  }
  fclose(fp);
  return (size_t)rss * (size_t)sysconf(_SC_PAGESIZE);

#else
  /* AIX, BSD, Solaris, and Unknown OS ------------------------ */
  return (size_t)0L; /* Unsupported. */
#endif
}

582
// -------------------------- concrete instantiations
583
584
template DLLExport string memToString<uint32_t>(const uint32_t);
template DLLExport string memToString<uint64_t>(const uint64_t);
585
// To initialize the static class variable.
586
std::mutex MemoryStats::mutexMemory;
587
588
589

} // namespace Kernel
} // namespace Mantid