Cycles: Avoid some expensive operations in header expansions

Basically gather lines as-is during traversal, avoiding allocating
memory for all the lines in headers.

Brings additional performance improvement abut 20%.

intern/cycles/util/util_path.cpp

@@ -771,12 +771,21 @@ bool path_remove(const string& path)
 
 struct SourceReplaceState {
 	typedef map<string, string> ProcessedMapping;
-
+	/* Base director for all relative include headers. */
 	string base;
+	/* Result of processed files. */
 	ProcessedMapping processed_files;
+	/* Set of files which are considered "precompiled" and which are replaced
+	 * with and empty string on a subsequent occurrence in include statement.
+	 */
 	set<string> precompiled_headers;
 };
 
+static string path_source_replace_includes_recursive(
+        const string& source,
+        const string& source_filepath,
+        SourceReplaceState *state);
+
 static string line_directive(const SourceReplaceState& state,
                              const string& path,
                              const int line)
@@ -798,6 +807,44 @@ static string line_directive(const SourceReplaceState& state,
 	return string_printf("#line %d \"%s\"", line, escaped_path.c_str());
 }
 
+static string path_source_handle_preprocessor(
+        const string& preprocessor_line,
+        const string& source_filepath,
+        const size_t line_number,
+        SourceReplaceState *state)
+{
+	string result = preprocessor_line;
+	string token = string_strip(
+	        preprocessor_line.substr(1, preprocessor_line.size() - 1));
+	if(string_startswith(token, "include")) {
+		token = string_strip(token.substr(7, token.size() - 7));
+		if(token[0] == '"') {
+			const size_t n_start = 1;
+			const size_t n_end = token.find("\"", n_start);
+			const string filename = token.substr(n_start, n_end - n_start);
+			const bool is_precompiled = string_endswith(token, "// PRECOMPILED");
+			string filepath = path_join(state->base, filename);
+			if(!path_exists(filepath)) {
+				filepath = path_join(path_dirname(source_filepath),
+				                     filename);
+			}
+			if(is_precompiled) {
+				state->precompiled_headers.insert(filepath);
+			}
+			string text;
+			if(path_read_text(filepath, text)) {
+				text = path_source_replace_includes_recursive(
+				        text, filepath, state);
+				/* Use line directives for better error messages. */
+				result = line_directive(*state, filepath, 1) + "\n"
+				     + text + "\n"
+				     + line_directive(*state, source_filepath, line_number + 1);
+			}
+		}
+	}
+	return result;
+}
+
 /* Our own little c preprocessor that replaces #includes with the file
  * contents, to work around issue of OpenCL drivers not supporting
  * include paths with spaces in them.
@@ -821,42 +868,42 @@ static string path_source_replace_includes_recursive(
 	}
 	/* Perform full file processing.  */
 	string result = "";
-	vector<string> lines;
+	const size_t source_length = source.length();
-	string_split(lines, source, "\n", false);
+	size_t index = 0;
-	for(size_t i = 0; i < lines.size(); ++i) {
+	size_t line_number = 0, column_number = 1;
-		const string& line = lines[i];
+	bool inside_preprocessor = false;
-		if(line[0] == '#') {
+	string preprocessor_line = "";
-			string token = string_strip(line.substr(1, line.size() - 1));
+	while(index < source_length) {
-			if(string_startswith(token, "include")) {
+		const char ch = source[index];
-				token = string_strip(token.substr(7, token.size() - 7));
+		if(ch == '\n') {
-				if(token[0] == '"') {
+			if(inside_preprocessor) {
-					const size_t n_start = 1;
+				result += path_source_handle_preprocessor(preprocessor_line,
-					const size_t n_end = token.find("\"", n_start);
+				                                          source_filepath,
-					const string filename = token.substr(n_start, n_end - n_start);
+				                                          line_number,
-					const bool is_precompiled = string_endswith(token, "// PRECOMPILED");
+				                                          state);
-					string filepath = path_join(state->base, filename);
-					if(!path_exists(filepath)) {
-						filepath = path_join(path_dirname(source_filepath),
-						                     filename);
-					}
-					if(is_precompiled) {
-						state->precompiled_headers.insert(filepath);
-					}
-					string text;
-					if(path_read_text(filepath, text)) {
-						text = path_source_replace_includes_recursive(
-						        text, filepath, state);
-						/* Use line directives for better error messages. */
-						result += line_directive(*state, filepath, 1)
-						     + token.replace(0, n_end + 1, "\n" + text + "\n")
-						     + line_directive(*state, source_filepath, i + 1) +
-						     "\n";
-						continue;
-					}
-				}
 			}
+			inside_preprocessor = false;
+			preprocessor_line = "";
+			column_number = 0;
+			++line_number;
 		}
-		result += line + "\n";
+		else if(ch == '#' && column_number == 1) {
+			inside_preprocessor = true;
+		}
+		if(inside_preprocessor) {
+			preprocessor_line += ch;
+		}
+		else {
+			result += ch;
+		}
+		++index;
+		++column_number;
+	}
+	if(inside_preprocessor) {
+		result += path_source_handle_preprocessor(preprocessor_line,
+		                                          source_filepath,
+		                                          line_number,
+		                                          state);
 	}
 	/* Store result for further reuse. */
 	state->processed_files[source_filepath] = result;