danny/emory
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

tools/analyse: use a deque to avoid expensive reallocations

Browse Source 
We can amass a fair amount of allocated memory on some operations, so a
vector resize can exceed the available memory of a system on some
operations.
This commit is contained in:
Danny Robson 2021-01-23 06:48:39 +10:00
parent 6d584b10ec
commit bdc4a09356
1 changed files with 9 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
tools/analyse.cpp
View File

@ -24,6 +24,7 @@
#include <filesystem> #include <filesystem>
#include <fstream> #include <fstream>
#include <iostream> #include <iostream>
#include <deque>
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
@ -64,7 +65,7 @@ static bool region_equal (
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
static void find_path_chunks ( static void find_path_chunks (
std::vector<emory::chunk::region> &res, std::deque<emory::chunk::region> &res,
std::filesystem::path const &src, std::filesystem::path const &src,
emory::chunk::params const &p emory::chunk::params const &p
); );
@ -75,7 +76,7 @@ static void find_path_chunks (
static static
void void
find_regular_chunks ( find_regular_chunks (
std::vector<emory::chunk::region> &res, std::deque<emory::chunk::region> &res,
std::filesystem::path const &src, std::filesystem::path const &src,
emory::chunk::params const &p emory::chunk::params const &p
) { ) {
@ -96,7 +97,7 @@ find_regular_chunks (
static static
void void
find_directory_chunks ( find_directory_chunks (
std::vector<emory::chunk::region> &res, std::deque<emory::chunk::region> &res,
std::filesystem::path const &src, std::filesystem::path const &src,
emory::chunk::params const &p emory::chunk::params const &p
) { ) {
@ -111,7 +112,7 @@ find_directory_chunks (
/// Scan chunks from a given path by dispatching to `find_foo_chunks` style /// Scan chunks from a given path by dispatching to `find_foo_chunks` style
/// functions depending on the file type. /// functions depending on the file type.
static void find_path_chunks ( static void find_path_chunks (
std::vector<emory::chunk::region> &res, std::deque<emory::chunk::region> &res,
std::filesystem::path const &src, std::filesystem::path const &src,
emory::chunk::params const &p emory::chunk::params const &p
) { ) {
@ -139,12 +140,12 @@ static void find_path_chunks (
///---------------------------------------------------------------------------- ///----------------------------------------------------------------------------
/// Find all regions in a path and return a vector of the regions. /// Find all regions in a path and return a container of the regions.
static static
std::vector<emory::chunk::region> std::deque<emory::chunk::region>
find_chunks (std::filesystem::path const &src, emory::chunk::params const &p) find_chunks (std::filesystem::path const &src, emory::chunk::params const &p)
{ {
std::vector<emory::chunk::region> res; std::deque<emory::chunk::region> res;
find_path_chunks (res, src, p); find_path_chunks (res, src, p);
return res; return res;
} }
@ -194,7 +195,7 @@ int main (int argc, char const **argv)
// Find all the chunks and prepare them for output // Find all the chunks and prepare them for output
std::cout << "processing\n"; std::cout << "processing\n";
std::vector<emory::chunk::region> src = find_chunks (argv[ARGS_INPUT], p); auto src = find_chunks (argv[ARGS_INPUT], p);
fmt::print ("analysing {} chunks\n", src.size ()); fmt::print ("analysing {} chunks\n", src.size ());
std::sort (src.begin (), src.end (), region_less); std::sort (src.begin (), src.end (), region_less);