github_mirrors/vectorscan
3
0
Fork 1
mirror of https://github.com/VectorCamp/vectorscan.git synced 2025-06-28 16:41:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

reduce memory use in ng_small_literal_set/ng_literal_decorated

Browse Source 
These passes kept temporary strings/paths alive longer than was needed which
lead to high memory usage during these passes in pathological cases.
This commit is contained in:
Alex Coyte 2015-10-29 14:35:02 +11:00 committed by Matthew Barr
parent 1afc591c30
commit 629be08683
2 changed files with 34 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/nfagraph/ng_literal_decorated.cpp
View File

@ -67,6 +67,7 @@ static
bool findPaths(const NGHolder &g, vector<Path> &paths) { bool findPaths(const NGHolder &g, vector<Path> &paths) {
vector<NFAVertex> order = getTopoOrdering(g); vector<NFAVertex> order = getTopoOrdering(g);
vector<size_t> read_count(num_vertices(g));
vector<vector<Path>> built(num_vertices(g)); vector<vector<Path>> built(num_vertices(g));
for (auto it = order.rbegin(); it != order.rend(); ++it) { for (auto it = order.rbegin(); it != order.rend(); ++it) {
@ -74,6 +75,11 @@ bool findPaths(const NGHolder &g, vector<Path> &paths) {
auto &out = built[g[v].index]; auto &out = built[g[v].index];
assert(out.empty()); assert(out.empty());
read_count[g[v].index] = out_degree(v, g);
DEBUG_PRINTF("setting read_count to %zu for %u\n",
read_count[g[v].index], g[v].index);
if (v == g.start || v == g.startDs) { if (v == g.start || v == g.startDs) {
out.push_back({v}); out.push_back({v});
continue; continue;
@ -94,6 +100,9 @@ bool findPaths(const NGHolder &g, vector<Path> &paths) {
continue; continue;
} }
assert(!built[g[u].index].empty());
assert(read_count[g[u].index]);
for (const auto &p : built[g[u].index]) { for (const auto &p : built[g[u].index]) {
out.push_back(p); out.push_back(p);
out.back().push_back(v); out.back().push_back(v);
@ -105,6 +114,13 @@ bool findPaths(const NGHolder &g, vector<Path> &paths) {
return false; return false;
} }
} }
read_count[g[u].index]--;
if (!read_count[g[u].index]) {
DEBUG_PRINTF("clearing %u as finished reading\n", g[u].index);
built[g[u].index].clear();
built[g[u].index].shrink_to_fit();
}
} }
} }

18
src/nfagraph/ng_small_literal_set.cpp
View File

@ -118,10 +118,15 @@ bool findLiterals(const NGHolder &g,
vector<NFAVertex> order = getTopoOrdering(g); vector<NFAVertex> order = getTopoOrdering(g);
vector<set<sls_literal>> built(num_vertices(g)); vector<set<sls_literal>> built(num_vertices(g));
vector<size_t> read_count(num_vertices(g));
for (auto it = order.rbegin(); it != order.rend(); ++it) { for (auto it = order.rbegin(); it != order.rend(); ++it) {
NFAVertex v = *it; NFAVertex v = *it;
set<sls_literal> &out = built[g[v].index]; set<sls_literal> &out = built[g[v].index];
read_count[g[v].index] = out_degree(v, g);
DEBUG_PRINTF("setting read_count to %zu for %u\n",
read_count[g[v].index], g[v].index);
assert(out.empty()); assert(out.empty());
if (v == g.start) { if (v == g.start) {
@ -149,7 +154,10 @@ bool findLiterals(const NGHolder &g,
} }
set<sls_literal> &in = built[g[u].index]; set<sls_literal> &in = built[g[u].index];
DEBUG_PRINTF("getting from %u (%zu reads to go)\n",
g[u].index, read_count[g[u].index]);
assert(!in.empty()); assert(!in.empty());
assert(read_count[g[u].index]);
for (const sls_literal &lit : in) { for (const sls_literal &lit : in) {
if (accept) { if (accept) {
@ -171,10 +179,18 @@ bool findLiterals(const NGHolder &g,
out.insert(lit.append((u8)c, nocase)); out.insert(lit.append((u8)c, nocase));
if (out.size() + literals->size() > MAX_LITERAL_SET_SIZE) { if (out.size() + literals->size() > MAX_LITERAL_SET_SIZE) {
DEBUG_PRINTF("too big %zu + %zu\n", out.size(),
literals->size());
return false; return false;
} }
} }
} }
read_count[g[u].index]--;
if (!read_count[g[u].index]) {
DEBUG_PRINTF("clearing %u as finished reading\n", g[u].index);
in.clear();
}
} }
} }
@ -206,6 +222,8 @@ bool handleSmallLiteralSets(RoseBuild &rose, const NGHolder &g,
return false; return false;
} }
DEBUG_PRINTF("looking for literals\n");
map<sls_literal, ue2::flat_set<ReportID>> literals; map<sls_literal, ue2::flat_set<ReportID>> literals;
if (!findLiterals(g, &literals)) { if (!findLiterals(g, &literals)) {
DEBUG_PRINTF(":(\n"); DEBUG_PRINTF(":(\n");