// CPreproc.cpp
// (C)2009-2012,2020; Boost license @ LICENSE.md

#include "CPreproc.hpp"

#include <limits.h>
#include <time.h>
#include <stdexcept>

#include "AtomicString.h"
#include "CSupport.hpp"
#ifndef BUILD_Z_CPP
#include "_CSupport3.hpp"
#endif
#include "C_PPDecimalInteger.hpp"
#include "CPUInfo.hpp"
#include "errors.hpp"
#include "errcount.hpp"
#include "filesystem.h"
#include "langroute.hpp"
#include "load_src.hpp"
#include "ParseTree.hpp"
#include "str_aux.h"
#include "type_system.hpp"
#include "weak_token.hpp"
#include "_version.h"
#include "Zaimoni.STL/POD.hpp"
#include "Zaimoni.STL/LexParse/Token.hpp"
#include "Zaimoni.STL/LexParse/LangConf.hpp"
#include "Zaimoni.STL/search.hpp"
#include "Zaimoni.STL/OS/mutex.hpp"
#include "Zaimoni.STL/Pure.C/format_util.h"
#include "Zaimoni.STL/Perl_localize.hpp"

#include "DebugCSupport.h"

#include "Zaimoni.STL/Pure.C/comptest.h"
#ifdef ZAIMONI_HAS_MICROSOFT_IO_H
#include <direct.h>
#include <io.h>

#ifndef F_OK
#define F_OK 0
#define R_OK 2
#define W_OK 4
// #define X_OK
#endif
#else
#include <unistd.h>	// for getcwd; POSIX
#endif

using namespace zaimoni;

// beginning of multilingual support
#define ERR_STR "error: "
#define WARN_STR "warning: "

// define this to hard-code an install path
//#define INSTALL_TO "\\CPP_App\\Z.C++"
// for Z.C++ 0.0.9 and earlier
// #define RELATIVE_PATH_TO_LIB ".." ZAIMONI_PATH_SEP
// current value
#ifndef RELATIVE_PATH_TO_LIB
#define RELATIVE_PATH_TO_ABOVE_LIB "." ZAIMONI_PATH_SEP
#endif

//! \todo this should be language-sensitive; LangConf not a good location
static const char* const fixed_system_include_search[] = {
#ifdef INSTALL_TO
	// C++-only install paths
	INSTALL_TO ZAIMONI_PATH_SEP "lib" ZAIMONI_PATH_SEP "zc++-" ZCC_VERSION ZAIMONI_PATH_SEP "include",
	INSTALL_TO ZAIMONI_PATH_SEP "lib" ZAIMONI_PATH_SEP "zc++" ZAIMONI_PATH_SEP "include",
#endif	
	// C++-only relative paths
	RELATIVE_PATH_TO_ABOVE_LIB "lib" ZAIMONI_PATH_SEP "zc++-" ZCC_VERSION ZAIMONI_PATH_SEP "include",
	RELATIVE_PATH_TO_ABOVE_LIB "lib" ZAIMONI_PATH_SEP "zc++" ZAIMONI_PATH_SEP "include",
#ifdef INSTALL_TO
	// install path
	INSTALL_TO ZAIMONI_PATH_SEP "lib" ZAIMONI_PATH_SEP "zcc-" ZCC_VERSION ZAIMONI_PATH_SEP "include",
	INSTALL_TO ZAIMONI_PATH_SEP "lib" ZAIMONI_PATH_SEP "zcc" ZAIMONI_PATH_SEP "include",
	INSTALL_TO ZAIMONI_PATH_SEP "include",
#endif	
	// relative paths
	RELATIVE_PATH_TO_ABOVE_LIB "lib" ZAIMONI_PATH_SEP "zcc-" ZCC_VERSION ZAIMONI_PATH_SEP "include",
	RELATIVE_PATH_TO_ABOVE_LIB "lib" ZAIMONI_PATH_SEP "zcc" ZAIMONI_PATH_SEP "include",
	RELATIVE_PATH_TO_ABOVE_LIB "include"
};

static const char* actual_system_include_search[] =	{
#ifdef INSTALL_TO
	// C++-only install paths
	NULL,
	NULL,
#endif	
	// C++-only relative paths
	NULL,
	NULL,
#ifdef INSTALL_TO
	// install path
	NULL,
	NULL,
	NULL,
#endif	
	// relative paths
	NULL,
	NULL,
	NULL
};

static_assert(STATIC_SIZE(fixed_system_include_search)==STATIC_SIZE(actual_system_include_search));

#ifdef INSTALL_TO
#define START_CPP_ONLY_PATHS 4
#else
#define START_CPP_ONLY_PATHS 2
#endif

static void init_fixed_system_include_search(void)
{
	static bool fixed_system_include_exists_init = false;

	if (fixed_system_include_exists_init) return;
	char filepath[FILENAME_MAX];
	size_t i = STATIC_SIZE(fixed_system_include_search);
	do	{
		--i;
		assert(!is_empty_string(fixed_system_include_search[i]));
		if ('.'==fixed_system_include_search[i][0])
			{	// obviously relative path
			auto test(std::filesystem::path(self_path().parent_path()) /= fixed_system_include_search[i]);
			try {
				auto canon = std::filesystem::canonical(test);
				actual_system_include_search[i] = register_string(canon.generic_string().c_str());
			} catch (std::filesystem::filesystem_error& e) {
				// doesn't really exist; omit
			}
			}
		else if (ZAIMONI_PATH_SEP[0]==fixed_system_include_search[i][0])
			{	// absolute path, current drive
			char* exists = z_realpath(filepath,fixed_system_include_search[i]);
			if (exists && !access(exists,F_OK))
				actual_system_include_search[i] = register_string(filepath);
			};
		}
	while(0<i);
	fixed_system_include_exists_init = true;
}

#define LEXER_STRICT_UB (Lexer::CPlusPlus+1)

/* C99
1 The following macro names151) shall be defined by the implementation:
_ _DATE_ _ The date of translation of the preprocessing translation unit: a character
string literal of the form "Mmm dd yyyy", where the names of the
months are the same as those generated by the asctime function, and the
first character of dd is a space character if the value is less than 10. If the
date of translation is not available, an implementation-defined valid date
shall be supplied.
_ _FILE_ _ The presumed name of the current source file (a character string literal).152)
_ _LINE_ _ The presumed line number (within the current source file) of the current
source line (an integer constant).152)
_ _STDC_ _ The integer constant 1, intended to indicate a conforming implementation.
_ _STDC_HOSTED_ _ The integer constant 1 if the implementation is a hosted
implementation or the integer constant 0 if it is not.
_ _STDC_VERSION_ _ The integer constant 199901L.153)
_ _TIME_ _ The time of translation of the preprocessing translation unit: a character
string literal of the form "hh:mm:ss" as in the time generated by the
asctime function. If the time of translation is not available, an
implementation-defined valid time shall be supplied.
2 The following macro names are conditionally defined by the implementation:
_ _STDC_IEC_559_ _ The integer constant 1, intended to indicate conformance to the
specifications in annex F (IEC 60559 floating-point arithmetic).

_ _STDC_IEC_559_COMPLEX_ _ The integer constant 1, intended to indicate
adherence to the specifications in informative annex G (IEC 60559
compatible complex arithmetic).
_ _STDC_ISO_10646_ _ An integer constant of the form yyyymmL (for example,
199712L). If this symbol is defined, then every character in the Unicode
required set, when stored in an object of type wchar_t, has the same
value as the short identifier of that character. The Unicode required set
consists of all the characters that are defined by ISO/IEC 10646, along with
all amendments and technical corrigenda, as of the specified year and
month.
3 The values of the predefined macros (except for _ _FILE_ _ and _ _LINE_ _) remain
constant throughout the translation unit.
4 None of these macro names, nor the identifier defined, shall be the subject of a
#define or a #undef preprocessing directive. Any other predefined macro names
shall begin with a leading underscore followed by an uppercase letter or a second
underscore.
5 The implementation shall not predefine the macro _ _cplusplus, nor shall it define it
in any standard header.
*/

/* C++0x
1 The following macro names shall be defined by the implementation:
_ _ cplusplus
The name _ _ cplusplus is defined to the value [tbd] when compiling a C++ translation unit.146)
_ _ DATE _ _
The date of translation of the source file (a character string literal of the form "Mmm dd yyyy", where the names
of the months are the same as those generated by the asctime function, and the first character of dd is a space
character if the value is less than 10). If the date of translation is not available, an implementation-defined valid
date is supplied.
_ _ FILE _ _
The presumed name of the source file (a character string literal).
_ _ LINE _ _
The line number of the current source line (a decimal constant).
_ _ STDC_HOSTED _ _
The integer constant 1 if the implementation is a hosted implementation or the integer constant 0 if it is not.
_ _ TIME _ _
The time of translation of the source file (a character string literal of the form "hh:mm:ss" as in the time generated
by the asctime function). If the time of translation is not available, an implementation-defined valid time is
supplied.
2 The following macro names are conditionally defined by the implementation:
_ _ STDC _ _
Whether _ _ STDC _ _ is predefined and if so, what its value is, are implementation-defined.

_ _ STDC_VERSION _ _
Whether _ _ STDC_VERSION _ _ is predefined and if so, what its value is, are implementation-defined.
_ _ STDC_ISO_10646 _ _
An integer constant of the form yyyymmL (for example, 199712L), intended to indicate that values of type
wchar_t are the coded representations of the characters defined by ISO/IEC 10646, along with all amendments
and technical corrigenda as of the specified year and month.

3 The values of the predefined macros (except for _ _ LINE _ _ and _ _ FILE _ _) remain constant throughout the translation
unit.
4 If any of the pre-defined macro names in this subclause, or the identifier defined, is the subject of a #define or
a #undef preprocessing directive, the behavior is undefined. Any other predefined macro names shall begin with a
leading underscore followed by an uppercase letter or a second underscore.
*/

#define DICT_STRUCT(A) { (A), sizeof(A)-1 }

// design decision: make this consistent across both C, C++
// design decision: make this consistent across all versions of C, C++
static const POD_pair<const char*,size_t> C99_CPP0x_locked_macros_default[]
 =	{	DICT_STRUCT("__DATE__"),				// determine in realtime at start of translation
		DICT_STRUCT("__TIME__"),				// determine in realtime at start of translation
		DICT_STRUCT("__FILE__"),				// special handling
		DICT_STRUCT("__LINE__"),				// special handling
		DICT_STRUCT("__VA_ARGS__"),				// disallowed as identifier anywhere except variadic function macro replacement list, C99/C++0x
		// __STDC_ macros are locked anyway, as they are reserved by the C/C++ standards
//		DICT_STRUCT("__STDC__"),				// define to 1 for both (GCC, note that K&R prototype-compatibility code uses this)
//		DICT_STRUCT("__STDC_HOSTED__"),			// define to 1 for now
//		DICT_STRUCT("__STDC_VERSION__"),		// define to 1 for now (as we don't really implement much of anything yet)
//		DICT_STRUCT("__STDC_IEC_559__"),		// undefined
//		DICT_STRUCT("__STDC_IEC_559_COMPLEX__"),// mentioned in C99 only; undefined
//		DICT_STRUCT("__STDC_IEC_10646__"),		// mentioned in C99 only; undefined
		DICT_STRUCT("__cplusplus"),				// undefined/C99, define to 1 for C++0x
		DICT_STRUCT("defined"),					// must be undefined or else undefined behavior, even not considering C99 6.10.8 p4
		DICT_STRUCT("_Pragma"),					// considering the syntactical role of the _Pragma operator, lock it even though the standards don't require us to. 
		DICT_STRUCT("__TIMESTAMP__"),			// lock down our extension macros
		DICT_STRUCT("__COUNTER__"),				// lock down our extension macros
		DICT_STRUCT("__INCLUDE_LEVEL__"),		// lock down our extension macros
		DICT_STRUCT("__has_include"),			// lock down our extension macros
		DICT_STRUCT("__ZCC__"),					// lock down our identity
		DICT_STRUCT("__ZCC_MINOR__"),			// lock down our identity
		DICT_STRUCT("__ZCC_PATCHLEVEL__")		// lock down our identity
	};

static const POD_pair<const char*,const char*> C99_macro_identifier_default[]
 =	{	{"__DATE__", NULL},			// start standard-mandated macros
		{"__TIME__", NULL},
		{"__FILE__", NULL},
		{"__LINE__", NULL},
		{"__STDC__", "1"},
		{"__STDC_HOSTED__", "1"},
		{"__STDC_VERSION__", "1"},		// end standard-mandated macros
		{"__TIMESTAMP__", NULL},		// extension: GCC, CLang, MSVC
		{"__COUNTER__", NULL},			// extension: GCC, CLang, MSVC
		{"__INCLUDE_LEVEL__", NULL},	// extension: GCC, CLang
		{"__ZCC__", DEEP_STRINGIZE(ZCC_VERSION_MAJOR)},				// identity : major version
		{"__ZCC_MINOR__", DEEP_STRINGIZE(ZCC_VERSION_MINOR)},		// minor version
		{"__ZCC_PATCHLEVEL__", DEEP_STRINGIZE(ZCC_VERSION_PATCH)}	// patchlevel
	};

static const POD_pair<const char*,const char*> CPP0x_macro_identifier_default[]
 =	{	{"__TIMESTAMP__", NULL},		// extension: GCC, CLang, MSVC
		{"__COUNTER__", NULL},			// extension: GCC, CLang, MSVC
		{"__INCLUDE_LEVEL__", NULL},	// extension: GCC, CLang
		{"__DATE__", NULL},			// start standard-mandated macros
		{"__TIME__", NULL},
		{"__FILE__", NULL},
		{"__LINE__", NULL},
		{"__STDC__", "1"},
		{"__STDC_HOSTED__", "1"},
		{"__STDC_VERSION__", "1"},
		{"__cplusplus", "1"},		// end standard-mandated macros
		{"__ZCC__", DEEP_STRINGIZE(ZCC_VERSION_MAJOR)},				// identity : major version
		{"__ZCC_MINOR__", DEEP_STRINGIZE(ZCC_VERSION_MINOR)},		// minor version
		{"__ZCC_PATCHLEVEL__", DEEP_STRINGIZE(ZCC_VERSION_PATCH)}	// patchlevel
	};

CPreprocessor::CPreprocessor(const virtual_machine::CPUInfo& _target_machine, const char* const _lang)
:	counter_macro(0),
	include_level(0),
	lang_code(lang_index(_lang)),
	lang(lexer_from_lang(lang_code)),
	target_machine(_target_machine),
	macro_identifier_default(NULL),
	macro_locked_default(C99_CPP0x_locked_macros_default),
	macro_identifier_default_count(0),
	macro_locked_default_count(STATIC_SIZE(C99_CPP0x_locked_macros_default)),
	debug_mode(false)
{
	switch(lang_code)
	{
	default: FATAL("Invalid language code");
	case Lang::C:
		macro_identifier_default = C99_macro_identifier_default;
		macro_identifier_default_count = STATIC_SIZE(C99_macro_identifier_default);
		break;
	case Lang::CPlusPlus:
		macro_identifier_default = CPP0x_macro_identifier_default;
		macro_identifier_default_count = STATIC_SIZE(CPP0x_macro_identifier_default);
//		break;
	}
	init_fixed_system_include_search();
	// reality checks on the language
	assert(lang.EscapeStringLength);
	assert(lang.EscapeString);
	assert(lang.UnescapeStringLength);
	assert(lang.UnescapeString);
	assert(lang.pp_support);
	assert(lang.InvariantKeywords);
	assert(0<lang.len_InvariantKeywords);
}

static const POD_pair<const char*,size_t> valid_directives[]
	=	{	DICT_STRUCT("if"),
			DICT_STRUCT("ifdef"),
			DICT_STRUCT("ifndef"),
			DICT_STRUCT("elif"),
			DICT_STRUCT("else"),
			DICT_STRUCT("endif"),
			DICT_STRUCT("include"),
			DICT_STRUCT("define"),
			DICT_STRUCT("undef"),
			DICT_STRUCT("line"),
			DICT_STRUCT("error"),
			DICT_STRUCT("pragma")
		};

static const POD_pair<const char*,size_t> accept_pragma_leading_tokens[]
	=	{	DICT_STRUCT("STDC"),	// C99
			DICT_STRUCT("ZCC"),		// our own
			DICT_STRUCT("GCC"),		// we also pay attention to GCC pragmas
			DICT_STRUCT("message")	// #pragma message("...")
		};

// yes, enum would be better style
#define PRAGMA_LEADING_STDC 0
#define PRAGMA_LEADING_ZCC 1
// #define PRAGMA_LEADING_GCC 2
#define PRAGMA_MESSAGE 3

static const POD_pair<const char*,size_t> pragma_STDC_on_off_switch[]	// C99 6.10.5p2
	=	{	DICT_STRUCT("OFF"),
			DICT_STRUCT("DEFAULT"),
			DICT_STRUCT("ON")
		};

#define ON_OFF_SWITCH_OFF 0
#define ON_OFF_SWITCH_DEFAULT 1
#define ON_OFF_SWITCH_ON 2

static const POD_pair<const char*,size_t> pragma_STDC_keywords[]
	=	{	DICT_STRUCT("FP_CONTRACT"),
			DICT_STRUCT("FENV_ACCESS"),
			DICT_STRUCT("CX_LIMITED_RANGE")
		};

#define PRAGMA_STDC_FP_CONTRACT 0
#define PRAGMA_STDC_FENV_ACCESS 1
#define PRAGMA_STDC_CX_LIMITED_RANGE 2

static const POD_pair<const char*,size_t> pragma_ZCC_keywords[]
	=	{	DICT_STRUCT("lock"),
			DICT_STRUCT("enable_typeid")
		};

#define PRAGMA_ZCC_LOCK 0
#define PRAGMA_ZCC_ENABLE_TYPEID 1
#undef DICT_STRUCT

static void _init_weak_token(weak_token& dest, const Token<char>& x,const POD_triple<size_t,size_t,lex_flags>& pretoken)
{
	dest.token.first = x.data()+pretoken.first;
	dest.token.second = pretoken.second;
	dest.flags = pretoken.third;
	dest.logical_line.first = x.original_line.first;
	dest.logical_line.second = x.original_line.second+pretoken.first;
#ifdef NO_LEGACY_FIELDS
	dest.src_filename = register_string(x.src->generic_string().c_str());
#else
	dest.src_filename = x.src_filename.data();
#endif
}

static void _weak_tokenize_aux(const Token<char>& x,const autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> >& pretokenized, autovalarray_ptr<weak_token>& weaktoken_list)
{
	assert(!x.empty());
	assert(!pretokenized.empty());
	weaktoken_list.resize(pretokenized.size());
	size_t j = pretokenized.size();
	do	{
		--j;
		_init_weak_token(weaktoken_list[j],x,pretokenized[j]);
		}
	while(0<j);
}

static void _parsetreeize_aux(const Token<char>& x,const autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> >& pretokenized, parse_tree& parsetree)
{
	assert(!x.empty());
	assert(!pretokenized.empty());
	if (1==pretokenized.size())
		{
		_init_weak_token(parsetree.index_tokens[0],x,pretokenized.front());
		return;
		}
	if (!parsetree.resize<0>(pretokenized.size())) throw std::bad_alloc();
	size_t j = pretokenized.size();
	do	{
		--j;
		_init_weak_token(parsetree.c_array<0>()[j]->index_tokens[0],x,pretokenized[j]);
		}
	while(0<j);
}

static void _flush_duplicated_ws(Token<char>& x, const LangConf& lang)
{
	lex_flags scratch_flags;
	size_t offset2 = lang.UnfilteredNextToken(x.data(),scratch_flags);
	while(offset2<x.size())
		{
		const size_t skip_ws = strspn(x.data()+offset2,lang.WhiteSpace);
		if (0<skip_ws)
			{
			if (skip_ws>=x.size()-offset2)
				{
				x.lslice(offset2);
				return;
				};
			if (strchr(lang.AtomicSymbols,x.data()[offset2-1]) || strchr(lang.AtomicSymbols,x.data()[offset2+skip_ws]))
				x.intradelete(offset2,skip_ws);
			else{
				x.replace_once(std::nothrow,offset2,skip_ws,' ');
				++offset2;
				}
			}
		offset2 += lang.UnfilteredNextToken(x.data()+offset2,scratch_flags);
		}
}

template<class IntType,class Iterator>
void
STL_translate_first(IntType origin, Iterator iter, const Iterator iter_end)
{
	while(iter_end!=iter) (iter++)->first += origin;
}

template<class IntType,class T>
inline void STL_translate_first(IntType origin, T& x)
{
	assert(!x.empty());
	STL_translate_first(origin,x.begin(),x.end());
}

template<class IntType,class Iterator>
void
STL_translate_second(IntType origin, Iterator iter, const Iterator iter_end)
{
	while(iter_end!=iter) (iter++)->second += origin;
}

//! \todo use lang.AtomicSymbols to get further compression
static bool _flush_token_gaps(Token<char>& x, autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> >& pretokenized, const LangConf& lang)
{
	assert(!x.empty());
	assert(!pretokenized.empty());
	bool acted = false;
	if (x.size()>pretokenized.back().first+pretokenized.back().second)
		{
		x.lslice(pretokenized.back().first+pretokenized.back().second);
		acted = true;
		};
	if (1<pretokenized.size())
		{
		size_t i = pretokenized.size()-1;
		bool old_atomic = 1==pretokenized[i].second && strchr(lang.AtomicSymbols,x.data()[pretokenized[i].first]);
		do	{
			--i;
			const bool new_atomic = 1==pretokenized[i].second && strchr(lang.AtomicSymbols,x.data()[pretokenized[i].first]);
			if (new_atomic || old_atomic)
				{
				if (pretokenized[i+1].first>pretokenized[i].first+pretokenized[i].second)
					{
					const size_t delta = pretokenized[i+1].first-(pretokenized[i].first+pretokenized[i].second);
					x.intradelete(pretokenized[i].first+pretokenized[i].second,delta);
					STL_translate_first(-delta,pretokenized.begin()+i+1,pretokenized.end());	// unsigned modulo arithmetic makes this work
					acted = true;
					}
				}
			else if (pretokenized[i+1].first-1>pretokenized[i].first+pretokenized[i].second)
				{
				const size_t delta = pretokenized[i+1].first-(pretokenized[i].first+pretokenized[i].second)-1;
				x.replace_once(std::nothrow,pretokenized[i].first+pretokenized[i].second,delta+1,' ');
				STL_translate_first(-delta,pretokenized.begin()+i+1,pretokenized.end());	// unsigned modulo arithmetic makes this work
				acted = true;
				};
			old_atomic = new_atomic;
			}
		while(0<i);
		}
	// do *not* remove space before the first token; CPreprocessor::if_elif_syntax_ok will break, as that's where the #if/#elif is
	return acted;
}

namespace PP {

enum directive_indexes	{	IF = 0,
							IFDEF,
							IFNDEF,
							ELIF,
							ELSE,
							ENDIF,
							INCLUDE,
							DEFINE,
							UNDEF,
							LINE,
							ERROR,
							PRAGMA
						};

}

#define MAX_PP_DIRECTIVE (PP::PRAGMA+1)
#define PP_INVALID 15

// we assume that it takes exactly 4 bits to numerically represent the above enum
// we assume that 15 is available to flag invalid
static_assert((1<<4)>=MAX_PP_DIRECTIVE && ((1<<3)<MAX_PP_DIRECTIVE));

static errr find_directive(const char* const Text, const LangConf& lang)
{
	assert(!is_empty_string(Text));
	size_t i = STATIC_SIZE(valid_directives);
	do	{
		const POD_pair<const char*,size_t>& x = valid_directives[--i]; 
		if (!strncmp(x.first,Text,x.second)
			&& (x.second==strlen(Text) || lang.IsWS(Text[x.second])))
			return i;
		}
	while(0<i);
	return -1;
}

static void message_header(const Token<char>& src)
{
	assert(!src.src_filename.empty() && *src.src_filename);
#ifdef NO_LEGACY_FIELDS
	message_header(src.src->filename().generic_string().c_str(), src.logical_line.first);
#else
	message_header(src.src_filename.data(),src.logical_line.first);
#endif
}

static void message_header2(const Token<char>& src,size_t where)
{
	assert(!src.src_filename.empty());
#ifdef NO_LEGACY_FIELDS
	INC_INFORM(src.src->filename().generic_string().c_str());
#else
	INC_INFORM(src.src_filename.data());
#endif
	INC_INFORM(':');
	INC_INFORM(src.logical_line.first);
	INC_INFORM('.');
	INC_INFORM(where);
	INC_INFORM(": ");
}

void CPreprocessor::detailed_UNICODE_syntax(Token<char>& x) const
{
	assert(!x.empty());
	const char* UNICODE_escape = strstr(x.data(),"\\U");
	while(NULL!=UNICODE_escape)
		{
		const size_t tail = x.size()-(UNICODE_escape-x.data());
		if (10U>tail)
			{	//! \test UNICODE.C99/Error_AstralTruncate.h
				//! \test UNICODE.C99/Error_AstralTruncate.hpp
			message_header(x);
			INC_INFORM(ERR_STR);
			INFORM("C99 6.4.3p1/C++98 2.2p2: astral UNICODE escape prematurely terminated by end of line");
			zcc_errors.inc_error();
			break;
			}
		if (!and_range_n(IsHexadecimalDigit,UNICODE_escape+2,8))
			{	//! \test UNICODE.C99/Error_AstralNonhex.h
				//! \test UNICODE.C99/Error_AstralNonhex.hpp
			message_header(x);
			INC_INFORM(ERR_STR);
			INFORM("C99 6.4.3p1/C++98 2.2p2: astral UNICODE escape contains non-hexadecimal digits");
			zcc_errors.inc_error();
			}
		else
			assert(4>std::count(UNICODE_escape+2,UNICODE_escape+6,'0'));	// should have caught this in pre-filtration
		UNICODE_escape = (12U<=tail) ? strstr(UNICODE_escape+10,"\\U") : NULL;
		};
	UNICODE_escape = strstr(x.data(),"\\u");
	while(NULL!=UNICODE_escape)
		{
		const size_t tail = x.size()-(UNICODE_escape-x.data());
		if (6U>tail)
			{	//! \test UNICODE.C99/Error_Truncate.h
				//! \test UNICODE.C99/Error_Truncate.hpp
				//! \test UNICODE.C99/Error_Truncate2.h
				//! \test UNICODE.C99/Error_Truncate2.hpp
			message_header(x);
			INC_INFORM(ERR_STR);
			INFORM("UNICODE escape prematurely terminated by end of line (C99 6.4.3p1/C++0x 2.2p2)");
			zcc_errors.inc_error();
			break;
			}
		if (!and_range_n(IsHexadecimalDigit,UNICODE_escape+2,4))
			{	//! \test UNICODE.C99/Error_Nonhex.h
				//! \test UNICODE.C99/Error_Nonhex.hpp
				//! \test UNICODE.C99/Error_Nonhex2.h
				//! \test UNICODE.C99/Error_Nonhex2.hpp
			message_header(x);
			INC_INFORM(ERR_STR);
			INFORM("UNICODE escape contains non-hexadecimal digits (C99 6.4.3p1/C++0x 2.2p2)");
			zcc_errors.inc_error();
			}
		else if ('D'==UNICODE_escape[2] && strchr(list_hexadecimal_digits+8,UNICODE_escape[3]))
			{	//! \test UNICODE.C99/Error_SZone.h
				//! \test UNICODE.C99/Error_SZone.hpp
				//! \test UNICODE.C99/Error_SZone2.h
				//! \test UNICODE.C99/Error_SZone2.hpp
			message_header(x);
			INC_INFORM(ERR_STR);
			INFORM("UNICODE escape is in S-zone (D800-DFFF), reserved for UTF-16 (C99 6.4.3p1/C++0x 2.2p2)");
			zcc_errors.inc_error();
			}
		else if (2==std::count(UNICODE_escape+2,UNICODE_escape+4,'0'))
			{	//! \todo fix assumption that CHAR_BIT==8
			const unsigned char tmp = (unsigned char)(16*InterpretHexadecimalDigit(UNICODE_escape[4])+InterpretHexadecimalDigit(UNICODE_escape[5]));
			bool down_convert = true;
			assert(160U>tmp);	// should have caught this already
			// C++98 is "anything goes", but don't worry about that for now
			// C99 and C++0x agree that these are bad
			if ((unsigned char)(32U)>tmp || ((unsigned char)('\x7F')<=tmp && (unsigned char)('\x9F')>=tmp))
				{	//! \test UNICODE.C99/Error_CTRL_low.h
					//! \test UNICODE.C99/Error_CTRL_low.hpp
					//! \test UNICODE.C99/Error_CTRL_low2.h
					//! \test UNICODE.C99/Error_CTRL_low2.hpp
					//! \test UNICODE.C99/Error_CTRL_high.h
					//! \test UNICODE.C99/Error_CTRL_high.hpp
					//! \test UNICODE.C99/Error_CTRL_high2.h
					//! \test UNICODE.C99/Error_CTRL_high2.hpp
				message_header(x);
				INC_INFORM(ERR_STR);
				INFORM("UNICODE escape is in a control character range (C99 6.4.3p1/C++0x 2.2p2)");
				zcc_errors.inc_error();
				down_convert = false;
				}
			else if (Lang::C==lang_code)
				{	// C rejects anything escapeish, or in the source character set
					//! \test UNICODE.C99/Error_Source.h
					//! \test UNICODE.C99/Error_Source2.h
				if ('$'!=tmp && '@'!=tmp && '`'!=tmp)	//! \todo deal with ASCII dependence
					{
					message_header(x);
					INC_INFORM(ERR_STR);
					INFORM("UNICODE escape is in the source character set (C99 6.4.3p1)");
					zcc_errors.inc_error();
					down_convert = false;
					}
				};
			if (down_convert)
				{	//! \test cpp/default/Preprocess_UNICODE_deastral.hpp, cpp/default/Preprocess_UNICODE_deastral.h
				const size_t offset = UNICODE_escape-x.data();
				x.replace_once(std::nothrow,offset,6,tmp);
				UNICODE_escape = (8U<=tail) ? strstr(x.data()+offset+1,"\\u") : NULL;
				continue;
				}
			}
		UNICODE_escape = (8U<=tail) ? strstr(UNICODE_escape+10,"\\u") : NULL;
		};
}

// flags type in Token is unsigned long
//! \todo consider hooking into POSIX (replace ULONG_BIT with LONG_BIT)
#define ULONG_BIT (sizeof(unsigned long)*CHAR_BIT)
#define PREPROCESSING_DIRECTIVE_FLAG (1UL<<(ULONG_BIT-1))
#define PACK_DIRECTIVE(A,B) {(A) &= ~((15UL)<<(ULONG_BIT-5)); (A) |= ((B)<<(ULONG_BIT-5));}
#define UNPACK_DIRECTIVE(A) (((A) & ~PREPROCESSING_DIRECTIVE_FLAG)>>(ULONG_BIT-5))
#define SYNTAX_CHECKED_FLAG (1UL<<(ULONG_BIT-6))
#define INVALID_DIRECTIVE_FLAG (1UL<<(ULONG_BIT-7))

/*! 
 * Preprocesses a file with the C preprocessor, using the lexer lang specified at preprocessor construction.
 * 
 * \param TokenList : tokens to be preprocessed.
 * 
 * \return bool
 */
bool CPreprocessor::preprocess(autovalarray_ptr<Token<char>* >& TokenList)
{
	autovalarray_ptr<char*> locked_macros;
	autovalarray_ptr<char*> macros_object;
	autovalarray_ptr<Token<char>*> macros_object_expansion;
	autovalarray_ptr<Token<char>*> macros_object_expansion_pre_eval;
	autovalarray_ptr<char*> macros_function;
	autovalarray_ptr<Token<char>*> macros_function_arglist;
	autovalarray_ptr<Token<char>*> macros_function_expansion;
	autovalarray_ptr<Token<char>*> macros_function_expansion_pre_eval;
	autovalarray_ptr<POD_triple<const char*, const char*,uintptr_t> > include_file_index;
	autovalarray_ptr<POD_pair<const char*,autovalarray_ptr<Token<char>*>* > > include_file_cache;
	type_system min_types((Lang::C==lang_code) ? C_atomic_types : CPP_atomic_types,(Lang::C==lang_code) ? C_TYPE_MAX : CPP_TYPE_MAX,C_int_priority+C_PP_INT_PRIORITY_ORIGIN,C_INT_PRIORITY_SIZE-C_PP_INT_PRIORITY_ORIGIN);

	// this is subject to the Y10K bug, per standard.
	// construct __DATE__, __TIME__ macro targets
	{
	time_date_buffer[0] = '"';
	time_buffer[0]='"';
	date_buffer[0]='"';
	time_t start_time = time(NULL);
	strcpy(time_date_buffer+1,ctime(&start_time));
	strcpy(time_date_buffer+1+24,"\"");
	// have base buffer, construct time
	strncpy(time_buffer+1,time_date_buffer+12,8);
	time_buffer[0]='"';
	strcpy(time_buffer+9,"\"");
	// construct date
	strncpy(date_buffer+1,time_date_buffer+5,7);
	strncpy(date_buffer+8,time_date_buffer+21,4);
	date_buffer[0]='"';
	strcpy(date_buffer+12,"\"");
	}

	{	// do detailed UNICODE error trapping here: S-zone [\uD800-\uDFFF], sub-\u00A0 processing (latter is language-sensitive), syntax
	const autovalarray_ptr<Token<char>* >::iterator iter_end = TokenList.end();
	autovalarray_ptr<Token<char>* >::iterator iter = TokenList.begin();
	while(iter!=iter_end) detailed_UNICODE_syntax(**iter++);
	}

	zaimoni::Perl::localize<type_system*> lock_types(parse_tree::types,&min_types);
	_preprocess(TokenList, locked_macros, macros_object, macros_object_expansion, macros_object_expansion_pre_eval, macros_function, macros_function_arglist, macros_function_expansion, macros_function_expansion_pre_eval, include_file_index, include_file_cache, min_types);

	// need whitespace tokens here to force pretty-printing
	debug_to_stderr(TokenList,macros_object,macros_object_expansion,macros_function,macros_function_arglist,macros_function_expansion,locked_macros);
	die_on_pp_errors();

	// invoke GC on include_file_cache
	size_t i = include_file_cache.size();
	while(0<i)
		{
		delete include_file_cache[--i].second;
		include_file_cache[i].second = NULL;	// for style reasons
		};
	include_file_cache.reset();	// remove
	include_file_index.reset();	// remove

	if (TokenList.empty())
		{	//! \todo make this more efficient by providing a global flush-all
		while(string_from_index(0)) deregister_index(0);
		return true;
		}

	// Reset internal flags
	// for this to work, #pragma must be converted to _Pragma operators
	// do not flush whitespace tokens here, as the standalone will need them
	{
	const autovalarray_ptr<Token<char>* >::iterator iter_end = TokenList.end();
	autovalarray_ptr<Token<char>* >::iterator iter = TokenList.begin();
	if (iter!=iter_end)
		do	{
			(*iter)->flags &= ~PREPROCESSING_DIRECTIVE_FLAG;
			PACK_DIRECTIVE((*iter)->flags,0);
			}
		while(++iter!=iter_end);
	}

	return true;
}

static inline bool
pragma_locked_macro(const char* const x,const size_t x_len,const autovalarray_ptr<char*>& locked_macros)
{
	assert(!is_empty_string(x));
	assert(0<x_len);
	return 0<=binary_find(x,x_len,locked_macros);
}

static bool line_is_preprocessing_directive(Token<char>& x)
{	// normalize leading %: to # to handle equivalency of these tokens
	//! \test cpp/Error_error_directive2.hpp
	if ((sizeof("%:")-1)<=x.size() && !strncmp(x.data(),"%:",sizeof("%:")-1)) x.replace_once(std::nothrow,0,sizeof("%:")-1,"#");
	return '#'==x.front();
}

static void discard_locked_macro(autovalarray_ptr<Token<char>* >& TokenList, const size_t i,const int directive_type,const size_t first_token_len)
{
	message_header(*TokenList[i]);
	INC_INFORM(ERR_STR);
	INC_INFORM("#");
	INC_INFORM(valid_directives[directive_type].first);
	INC_INFORM(" applied to locked macro ");
	INC_INFORM(TokenList[i]->data()+valid_directives[directive_type].second+2,first_token_len);
	INFORM("; discarding directive. (ZCPP pragma effect)");
	TokenList.DeleteIdx(i);
	zcc_errors.inc_error();
}

static void C99_reject_keyword_macros(autovalarray_ptr<Token<char>* >& TokenList, size_t include_where, const char* look_for, const LangConf& lang, autovalarray_ptr<char*>& macros_object, autovalarray_ptr<Token<char>*>& macros_object_expansion, autovalarray_ptr<Token<char>*>& macros_object_expansion_pre_eval, autovalarray_ptr<char*>& macros_function, autovalarray_ptr<Token<char>*>& macros_function_arglist, autovalarray_ptr<Token<char>*>& macros_function_expansion, autovalarray_ptr<Token<char>*>& macros_function_expansion_pre_eval)
{
	assert(NULL!=look_for);
	size_t j = lang.len_InvariantKeywords;
	do	{
		--j;
		const errr object_macro_index = binary_find(lang.InvariantKeywords[j].first,lang.InvariantKeywords[j].second,macros_object);
		const errr function_macro_index = binary_find(lang.InvariantKeywords[j].first,lang.InvariantKeywords[j].second,macros_function);
		assert(0>object_macro_index || 0>function_macro_index);
		if (0<=object_macro_index || 0<=function_macro_index)
			{
			message_header(*TokenList[include_where]);
			INC_INFORM(ERR_STR);
			INC_INFORM("keyword '");
			INC_INFORM(0<=object_macro_index ? macros_object[object_macro_index] : macros_function[function_macro_index]);
			INC_INFORM("' defined as a macro when including system header <");
			INC_INFORM(look_for);
			INFORM(">.  Defining undefined behavior as undefining macro and proceeding.  (C99 7.1.2p4)");
			zcc_errors.inc_error();
			if (0<=object_macro_index)
				{
				macros_object.DeleteIdx(object_macro_index);
				macros_object_expansion.DeleteIdx(object_macro_index);
				macros_object_expansion_pre_eval.DeleteIdx(object_macro_index);
				};
			if (0<=function_macro_index)
				{
				macros_function.DeleteIdx(function_macro_index);
				macros_function_arglist.DeleteIdx(function_macro_index);
				macros_function_expansion.DeleteIdx(function_macro_index);
				macros_function_expansion_pre_eval.DeleteIdx(function_macro_index);
				};
			return;
			}
		}
	while(0<j);
}

/*! 
 * returns code for hardcoded system header not in the file system
 * 
 * \param look_for header name
 * \param lang_code language code from preprocessor object
 * 
 * \return unsigned int
 */
static unsigned int
detect_hardcoded_system_header(const char* const look_for,size_t lang_code)
{
	switch(lang_code)
	{
	case Lang::CPlusPlus:	// C++ hard-coded system headers
		if (!strcmp(look_for,"climits")) return 1;
		if (!strcmp(look_for,"cstddef")) return 2;
		if (!strcmp(look_for,"cstdint")) return 3;
	case Lang::C:	// C, C++ hard-coded system headers
		if (!strcmp(look_for,"limits.h")) return 1;
		if (!strcmp(look_for,"stddef.h")) return 2;
		if (!strcmp(look_for,"stdint.h")) return 3;
	}
	return 0;
}

void
CPreprocessor::_preprocess(autovalarray_ptr<Token<char>* >& TokenList, autovalarray_ptr<char*>& locked_macros, autovalarray_ptr<char*>& macros_object, autovalarray_ptr<Token<char>*>& macros_object_expansion, autovalarray_ptr<Token<char>*>& macros_object_expansion_pre_eval, autovalarray_ptr<char*>& macros_function, autovalarray_ptr<Token<char>*>& macros_function_arglist, autovalarray_ptr<Token<char>*>& macros_function_expansion, autovalarray_ptr<Token<char>*>& macros_function_expansion_pre_eval, autovalarray_ptr<POD_triple<const char*, const char*,uintptr_t> >& include_file_index, autovalarray_ptr<POD_pair<const char*,autovalarray_ptr<Token<char>*>* > >& include_file_cache, const type_system& min_types)
{
Restart:
	size_t i = 0;
RestartAfterInclude:
	if (TokenList.empty()) return;
	size_t if_depth = 0;
	size_t if_where = 0;			// origin 0
	size_t else_where = 0;			// origin 1; 0 is not triggered
	size_t include_where = 0;		// origin 1; 0 is not triggered
	size_t restart_full_scan = 0;	// origin 1; 0 is not triggered
	do	{
		if (line_is_preprocessing_directive(*TokenList[i]))
			{	// directive
			{	// scoping brace
			const size_t token_size = TokenList[i]->size();
			if (1==token_size)
				{
				TokenList.DeleteIdx(i);	// Null directive #, delete
				if (0==i) goto Restart;
				--i;
				continue;
				};
			const size_t whitespace_size = strspn(TokenList[i]->data()+1,lang.WhiteSpace+1);
			if (token_size-1 == whitespace_size)
				{
				TokenList.DeleteIdx(i);	// Null directive #, delete
				if (0==i) goto Restart;
				--i;
				continue;
				};

			TokenList[i]->flags |= PREPROCESSING_DIRECTIVE_FLAG;
			if (0<whitespace_size)
				{	// normalize
				TokenList[i]->c_array()[whitespace_size] = '#';
				TokenList[i]->ltrim(whitespace_size);
				assert(!TokenList[i]->empty());
				assert('#'==TokenList[i]->front());
				};
			}	// end scoping brace
			const errr directive_type = find_directive(TokenList[i]->data()+1,lang);
			if (0>directive_type && 0==if_depth)
				{	//! \test Warn_unrecognized_directive.hpp : naked unrecognized directive
				message_header(*TokenList[i]);
				INC_INFORM("warning: unrecognized preprocessing directive '");
				size_t j = 0;
				while(++j<TokenList[i]->size() && !lang.IsWS_NotFirst(TokenList[i]->data()[j]))
					INC_INFORM(TokenList[i]->data()[j]);
				INFORM("' (C99 6.10p1/C++98 16.1p1)");
				if (bool_options[boolopt::warnings_are_errors]) zcc_errors.inc_error();
				TokenList.DeleteIdx(i);
				if (0==i) goto Restart;
				--i;
				continue;
				};

			// stripping trailing spaces damages incomplete string and character literals.
			switch(directive_type)
			{
			case PP::IF:		
			case PP::IFDEF:		
			case PP::IFNDEF:	
			case PP::ELIF:		
			case PP::ELSE:		
			case PP::ENDIF:		
			case PP::INCLUDE:	
			case PP::DEFINE:	
			case PP::UNDEF:		
			case PP::LINE:		
			case PP::ERROR:		
			case PP::PRAGMA:	{	// normalize the whitespace; allowed to shove directive left to do it
								size_t critical_offset = valid_directives[directive_type].second+1;
								if (TokenList[i]->size()>critical_offset)
									{
									const size_t intra_WS = strspn(TokenList[i]->data()+critical_offset,lang.WhiteSpace+1);
									if (0==intra_WS)
										//! \todo really should be a bit more clever about this
										TokenList[i]->replace_once(critical_offset,0,' ');
									else if (TokenList[i]->size()-critical_offset<=intra_WS)
										{
										if (PP::PRAGMA==directive_type)
											{
											TokenList.DeleteIdx(i);	// empty pragma is legal, but Z.C++ says it has no effect
											if (0==i) goto Restart;
											--i;
											continue;
											};
										TokenList[i]->rtrim(intra_WS);
										}
									else if (1<intra_WS)
										TokenList[i]->replace_once(std::nothrow,critical_offset,intra_WS,' ');
									else
										TokenList[i]->c_array()[critical_offset] = ' ';
									}
								PACK_DIRECTIVE(TokenList[i]->flags,directive_type);
								assert(TokenList[i]->size()>=valid_directives[directive_type].second+1);
								break;
								};
			default:			{	// Unrecognized.  Warn-and-delete when we know it's going to be executed (not within any conditional blocks.
								PACK_DIRECTIVE(TokenList[i]->flags,PP_INVALID);
								break;
								};
			};

			// #if nesting-depth 0 processing
			if (0==if_depth)
				{
				lex_flags first_token_flags = 0;
				size_t first_token_len = 0;
				if (PP::ERROR==directive_type)
					{	// an error by fiat
						//! \test cpp/Error_error_directive.hpp
						//! \test cpp/Error_error_directive2.hpp
					message_header(*TokenList[i]);
					INC_INFORM(ERR_STR);
					INFORM(TokenList[i]->data()+(sizeof("#error ")-1));
					TokenList.DeleteIdx(i);
					zcc_errors.inc_error();
					if (0==i) goto Restart;
					--i;
					continue;
					};
				if (   PP::ELSE ==directive_type
					|| PP::ELIF ==directive_type
					|| PP::ENDIF==directive_type)
					{	// no prior #if/#ifdef/#ifndef is an error
					//! \test Error_naked_else.hpp : #else without #if/#ifdef/#ifndef
					//! \test Error_naked_elif.hpp : #elif without #if/#ifdef/#ifndef
					//! \test Error_naked_endif.hpp : #endif without #if/#ifdef/#ifndef
					//! \test Error_elif_after_else.hpp : #elif after #else
					//! \test Error_else_after_else.hpp : #else after #else
					message_header(*TokenList[i]);
					INC_INFORM(ERR_STR);
					INC_INFORM("#");
					INC_INFORM(valid_directives[directive_type].first);
					INFORM(" without paired #if/#ifdef/#ifndef (C99 6.10p1/C++98 16.1p1)");
					TokenList.DeleteIdx(i);
					zcc_errors.inc_error();
					if (0==i) goto Restart;
					--i;
					continue;
					};
				if (   PP::DEFINE ==directive_type
					|| PP::UNDEF  ==directive_type
					|| PP::INCLUDE==directive_type
					|| PP::LINE   ==directive_type)
					{	// these must have preprocessing tokens to be meaningful
					//! \test Error_no_token_define.hpp : #define no tokens
					//! \test Error_no_token_undef.hpp : #undef no tokens
					//! \test Error_no_token_include.hpp : #include no tokens
					//! \test Error_no_token_line.hpp : #line no tokens
					if (TokenList[i]->size()==valid_directives[directive_type].second+1)
						{
						message_header(*TokenList[i]);
						INC_INFORM(ERR_STR);
						INC_INFORM("#");
						INC_INFORM(valid_directives[directive_type].first);
						INFORM(" must have preprocessing tokens afterwards (allowing for macro substitution); discarding. (C99 6.10p1/C++98 16.1p1)");
						TokenList.DeleteIdx(i);
						zcc_errors.inc_error();
						if (0==i) goto Restart;
						--i;
						continue;
						}
					}
				if (   PP::DEFINE ==directive_type
					|| PP::UNDEF  ==directive_type)
					{	// do basic legality checks now: C identifier, not locked
					const size_t critical_offset = valid_directives[directive_type].second+2;
					if (TokenList[i]->size()>critical_offset) first_token_len = lang.UnfilteredNextToken(TokenList[i]->data()+critical_offset,first_token_flags);
					if (C_TESTFLAG_IDENTIFIER!=first_token_flags)
						{	//! \test Error_undef_no_identifier.hpp : #undef without identifier
						message_header(*TokenList[i]);
						INC_INFORM(ERR_STR);
						INC_INFORM("#");
						INC_INFORM(valid_directives[directive_type].first);
						INFORM(" does not have an identifier afterwards; discarding. (C99 6.10p1/C++98 16.1p1)");
						TokenList.DeleteIdx(i);
						zcc_errors.inc_error();
						if (0==i) goto Restart;
						--i;
						continue;
						}

					// want to catch illegal tokens before ditching a #undef of a locked macro
					//! \test Error_undef_illegal_tokens.hpp : #undef illegal tokens
					if (PP::UNDEF==directive_type) truncate_illegal_tokens(*TokenList[i],directive_type,critical_offset+first_token_len);

					if (hard_locked_macro(TokenList[i]->data()+critical_offset,first_token_len))
						{	//! \test Error_undef_STDC.hpp : #undef __STDC__
						discard_locked_macro(TokenList,i,directive_type,first_token_len);
						if (0==i) goto Restart;
						--i;
						continue;
						};
					if (	0==include_where && 0==restart_full_scan
						&&	pragma_locked_macro(TokenList[i]->data()+critical_offset,first_token_len,locked_macros))
						{	// here so we don't recalculate the above
						//! \test cpp/Error_undef_locked_macro.hpp
						discard_locked_macro(TokenList,i,directive_type,first_token_len);
						if (0==i) goto Restart;
						--i;
						continue;
						}
					if (PP::DEFINE==directive_type && TokenList[i]->size()>critical_offset+first_token_len)
						{	// standardize silently
						const size_t intra_WS = strspn(TokenList[i]->data()+critical_offset+first_token_len,lang.WhiteSpace+1);
						if (TokenList[i]->size()-(critical_offset+first_token_len)<=intra_WS)
							TokenList[i]->rtrim(intra_WS);	// normalize -- null-def object-like macro
						else if (1<intra_WS)
							TokenList[i]->replace_once(std::nothrow,critical_offset+first_token_len,intra_WS,' ');
						}
					};
				if (PP::PRAGMA==directive_type)
					{
					const size_t critical_offset = valid_directives[directive_type].second+2;
					assert(TokenList[i]->size()>critical_offset);
					if (C99_VA_ARGS_flinch(*TokenList[i],sizeof("pragma ")-1))
						{	//! \test Error_pragma_VA_ARGS.hpp
						message_header(*TokenList[i]);
						INFORM("discarding pragma");
						TokenList.DeleteIdx(i);	// ignore the pragma
						if (0==i) goto Restart;
						--i;
						continue;
						};
					first_token_len = lang.UnfilteredNextToken(TokenList[i]->data()+critical_offset,first_token_flags);
					if (0>linear_find_lencached(TokenList[i]->data()+critical_offset, first_token_len, accept_pragma_leading_tokens, STATIC_SIZE(accept_pragma_leading_tokens)))
						{
						TokenList.DeleteIdx(i);	// ignore the pragma
						if (0==i) goto Restart;
						--i;
						continue;
						};
					}
				if (PP::INCLUDE==directive_type && 0==include_where)
					{	// we have an include; postpone inclusion
						// don't think memory conservation is ISO
					if (0==restart_full_scan)
						intradirective_preprocess(*TokenList[i], sizeof("#include ")-1,macros_object,macros_object_expansion,macros_function,macros_function_arglist,macros_function_expansion_pre_eval);
					include_where = i+1;
					};
				if (0==include_where && 0==restart_full_scan)
					{
					if (PP::LINE==directive_type)
						{
						if (C99_VA_ARGS_flinch(*TokenList[i],sizeof("#line ")-1))
							{	//! \test cpp/line.C99/Error_VA_ARGS.h, cpp/line.C99/Error_VA_ARGS.hpp
							message_header(*TokenList[i]);
							INFORM("discarding #line directive");
							TokenList.DeleteIdx(i);
							if (0==i) goto Restart;
							--i;
							continue;
							};

						intradirective_preprocess(*TokenList[i], sizeof("#line ")-1,macros_object,macros_object_expansion,macros_function,macros_function_arglist,macros_function_expansion_pre_eval);
						use_line_directive_and_discard(TokenList,i);
						if (0==i) goto Restart;
						--i;
						continue;
						};
					if (PP::UNDEF==directive_type)
						{
						const size_t critical_offset = valid_directives[directive_type].second+2;
						assert(0<first_token_len);
						assert(TokenList[i]->size()>=critical_offset+first_token_len);
						// apply undef
						const errr object_macro_index = binary_find(TokenList[i]->data()+critical_offset,first_token_len,macros_object);
						const errr function_macro_index = binary_find(TokenList[i]->data()+critical_offset,first_token_len,macros_function);
						assert(0>object_macro_index || 0>function_macro_index);
						if (0<=object_macro_index)
							{
							macros_object.DeleteIdx(object_macro_index);
							macros_object_expansion.DeleteIdx(object_macro_index);
							macros_object_expansion_pre_eval.DeleteIdx(object_macro_index);
							};
						if (0<=function_macro_index)
							{
							macros_function.DeleteIdx(function_macro_index);
							macros_function_arglist.DeleteIdx(function_macro_index);
							macros_function_expansion.DeleteIdx(function_macro_index);
							macros_function_expansion_pre_eval.DeleteIdx(function_macro_index);
							}
						TokenList.DeleteIdx(i);
						if (0==i) goto Restart;
						--i;
						continue;
						}
					if (PP::DEFINE==directive_type)
						{
						const size_t critical_offset = valid_directives[directive_type].second+2;
						assert(0<first_token_len);
						assert(TokenList[i]->size()>=critical_offset+first_token_len);
						const errr object_macro_index = binary_find(TokenList[i]->data()+critical_offset,first_token_len,macros_object);
						const errr function_macro_index = binary_find(TokenList[i]->data()+critical_offset,first_token_len,macros_function);
						assert(0>object_macro_index || 0>function_macro_index);
						if (TokenList[i]->size()==critical_offset+first_token_len)
							{	// object-like, empty string
ObjectLikeMacroEmptyString:
							if (	 0<=function_macro_index
								|| 	(0<=object_macro_index && NULL!=macros_object_expansion[object_macro_index]))
								{	//! \test define.C99/Error_dup1.hpp, define.C99/Error_dup1.h
									//! \test define.C99/Error_dup2.hpp, define.C99/Error_dup2.h
								discard_duplicate_define(TokenList,i,critical_offset,first_token_len);
								if (0==i) goto Restart;
								--i;
								continue;
								};
							if (0<=object_macro_index)
								{	// already present: discard silently
									//! \test define.C99/Pass_dup1.hpp, define.C99/Pass_dup1.h
								TokenList.DeleteIdx(i);	// discard silently
								if (0==i) goto Restart;
								--i;
								continue;
								};
							const size_t object_macro_insertion_index = BINARY_SEARCH_DECODE_INSERTION_POINT(object_macro_index);
							macros_object.insertNSlotsAt(1,object_macro_insertion_index);
							macros_object_expansion.insertNSlotsAt(1,object_macro_insertion_index);
							macros_object_expansion_pre_eval.insertNSlotsAt(1,object_macro_insertion_index);
							macros_object[object_macro_insertion_index] = C_make_string(TokenList[i]->data()+critical_offset,first_token_len);
							}
						else if (lang.IsWS_NotFirst(TokenList[i]->data()[critical_offset+first_token_len]))
							{	// object-like, not empty string
							if (	0<=function_macro_index
								|| 	(0<=object_macro_index && NULL==macros_object_expansion[object_macro_index]))
								{	//! \test define.C99/Error_dup3.hpp, define.C99/Error_dup3.h
									//! \test define.C99/Error_dup4.hpp, define.C99/Error_dup4.h
									//! \test define.C99/Pass_dup2.hpp, define.C99/Pass_dup2.h
								discard_duplicate_define(TokenList,i,critical_offset,first_token_len);
								if (0==i) goto Restart;
								--i;
								continue;
								}
							Token<char> expansion(*TokenList[i],critical_offset+first_token_len,TokenList[i]->size()-(critical_offset+first_token_len),0);
							normalize_macro_expansion(expansion,*TokenList[i],critical_offset,first_token_len);
							//! \test define.C99/Pass_dup2.hpp, define.C99/Pass_dup2.h
							//! \test define.C99/Error_concatenate1.hpp, define.C99/Error_concatenate1.h
							//! \test define.C99/Error_concatenate2.hpp, define.C99/Error_concatenate2.h
							if (discard_leading_trailing_concatenate_op(expansion))
								goto ObjectLikeMacroEmptyString;
							if (0<=object_macro_index)
								{	//! \test define.C99/Error_dup5.hpp, define.C99/Error_dup5.h
									//! \test define.C99/Pass_dup2.hpp, define.C99/Pass_dup2.h
								if (strcmp(expansion.data(),macros_object_expansion[object_macro_index]->data()))
									discard_duplicate_define(TokenList,i,critical_offset,first_token_len);
								else
									TokenList.DeleteIdx(i);	// discard silently even if it matches
								if (0==i) goto Restart;
								--i;
								continue;
								};
							if (C99_VA_ARGS_flinch(expansion,0))
								{	//! \test define.C99/Error_VA_ARGS.hpp, define.C99/Error_VA_ARGS.h
								message_header(expansion);
								INFORM("discarding macro");
								TokenList.DeleteIdx(i);
								if (0==i) goto Restart;
								--i;
								continue;
								}

							// DO NOT check for context free errors here; could legitimately want to deep-stringize every invocation of the macro
							const size_t object_macro_insertion_index = BINARY_SEARCH_DECODE_INSERTION_POINT(object_macro_index);
							macros_object.insertNSlotsAt(1,object_macro_insertion_index);
							macros_object_expansion.insertNSlotsAt(1,object_macro_insertion_index);
							macros_object_expansion_pre_eval.insertNSlotsAt(1,object_macro_insertion_index);
							macros_object[object_macro_insertion_index] = C_make_string(TokenList[i]->data()+critical_offset,first_token_len);
							
							macros_object_expansion_pre_eval[object_macro_insertion_index] = new Token<char>(expansion);
							macros_object_expansion[object_macro_insertion_index] = new Token<char>;
							*macros_object_expansion[object_macro_insertion_index] = std::move(expansion);
							object_macro_concatenate(*macros_object_expansion_pre_eval[object_macro_insertion_index]);
							if (C99_VA_ARGS_flinch(*macros_object_expansion_pre_eval[object_macro_insertion_index],0))
								{	//! \test define.C99/Error_concatenate5.hpp, define.C99/Error_concatenate5.h
								message_header(*macros_object_expansion_pre_eval[object_macro_insertion_index]);
								INFORM("discarding macro");
								macros_object.DeleteIdx(object_macro_insertion_index);
								macros_object_expansion.DeleteIdx(object_macro_insertion_index);
								macros_object_expansion_pre_eval.DeleteIdx(object_macro_insertion_index);
								TokenList.DeleteIdx(i);
								if (0==i) goto Restart;
								--i;
								continue;
								};
							//! \todo should do locked macros [not __FILE__ or __LINE__] that are not first or last identifiers on speculation
							}
						else if ('('==TokenList[i]->data()[critical_offset+first_token_len])
							{	// function-like
							const size_t argspan = function_macro_argument_span(TokenList[i]->data()+critical_offset+first_token_len);
							if (0==argspan)
								{
								message_header(*TokenList[i]);
								INC_INFORM(ERR_STR);
								INC_INFORM(TokenList[i]->data()+critical_offset,first_token_len);
								INFORM(" has an invalid argument list.  Discarding. (C99 6.10.3p/C++0x 16.3p10)");
								TokenList.DeleteIdx(i);
								zcc_errors.inc_error();
								if (0==i) goto Restart;
								--i;
								continue;
								};
							if (0<=object_macro_index)
								{	//! \test define.C99/Error_dup6.hpp, define.C99/Error_dup6.h
								discard_duplicate_define(TokenList,i,critical_offset,first_token_len);
								if (0==i) goto Restart;
								--i;
								continue;
								};
							Token<char> arglist(*TokenList[i],critical_offset+first_token_len,argspan,0);
							normalize_macro_expansion(arglist,*TokenList[i],critical_offset,first_token_len);	// should be no string literals here, so should be no errors here
							if (0<=function_macro_index && strcmp(arglist.data(),macros_function_arglist[function_macro_index]->data()))
								{	//! \test define.C99/Error_dup7.hpp, define.C99/Error_dup7.h
								discard_duplicate_define(TokenList,i,critical_offset,first_token_len);
								if (0==i) goto Restart;
								--i;
								continue;
								};
							if (TokenList[i]->size()-(critical_offset+first_token_len)>argspan)
								{
								const size_t skip_ws = strspn(TokenList[i]->data()+critical_offset+first_token_len+argspan,lang.WhiteSpace+1);
								if (TokenList[i]->size()-(critical_offset+first_token_len+argspan)<=skip_ws)
									TokenList[i]->rtrim(skip_ws);
								};
							if (TokenList[i]->size()-(critical_offset+first_token_len)<=argspan)
								{	// empty expansion
FunctionLikeMacroEmptyString:	if (0<=function_macro_index)
									{	//! \test define.C99/Error_dup9.hpp, define.C99/Error_dup9.h
										//! \test define.C99/Pass_dup4.h, define.C99/Pass_dup4.h
									if (NULL!=macros_function_expansion[function_macro_index])
										discard_duplicate_define(TokenList,i,critical_offset,first_token_len);
									else
										TokenList.DeleteIdx(i);	// ignore silently if exactly agreed
									if (0==i) goto Restart;
									--i;
									continue;
									}
								const size_t function_macro_insertion_index = BINARY_SEARCH_DECODE_INSERTION_POINT(function_macro_index);
								macros_function.insertNSlotsAt(1,function_macro_insertion_index);
								macros_function_arglist.insertNSlotsAt(1,function_macro_insertion_index);
								macros_function_expansion.insertNSlotsAt(1,function_macro_insertion_index);
								macros_function_expansion_pre_eval.insertNSlotsAt(1,function_macro_insertion_index);
								macros_function[function_macro_insertion_index] = C_make_string(TokenList[i]->data()+critical_offset,first_token_len);
								
								macros_function_arglist[function_macro_insertion_index] = new Token<char>;
								*macros_function_arglist[function_macro_insertion_index] = std::move(arglist);

								TokenList.DeleteIdx(i);
								if (0==i) goto Restart;
								--i;
								continue;
								};
							Token<char> expansion(*TokenList[i],critical_offset+first_token_len+argspan,TokenList[i]->size()-(critical_offset+first_token_len+argspan),0);
							normalize_macro_expansion(expansion,*TokenList[i],critical_offset,first_token_len);
							// white-box test policy: cases above will work
							if (discard_leading_trailing_concatenate_op(expansion))
								goto FunctionLikeMacroEmptyString;
							if (0<=function_macro_index)
								{	//! \test define.C99/Error_dup8.hpp, define.C99/Error_dup8.h
									//! \test define.C99/Pass_dup3.h, define.C99/Pass_dup3.h
								if (strcmp(expansion.data(),macros_function_expansion[function_macro_index]->data()))
									discard_duplicate_define(TokenList,i,critical_offset,first_token_len);
								else
									TokenList.DeleteIdx(i);	// discard silently even if it matches
								if (0==i) goto Restart;
								--i;
								continue;
								};
							if ((5>arglist.size() || strcmp(arglist.end()-(sizeof("...)")-1),"...)")) && C99_VA_ARGS_flinch(expansion,0))
								{	//! \test define.C99/Error_VA_ARGS2.hpp, define.C99/Error_VA_ARGS2.h
									//! \test define.C99/Pass_VA_ARGS.hpp, define.C99/Pass_VA_ARGS.h
								message_header(expansion);
								INFORM("discarding macro");
								TokenList.DeleteIdx(i);
								if (0==i) goto Restart;
								--i;
								continue;
								}

							// DO NOT check for context free errors here; could legitimately want to deep-stringize every invocation of the macro
							const size_t function_macro_insertion_index = BINARY_SEARCH_DECODE_INSERTION_POINT(function_macro_index);
							macros_function.insertNSlotsAt(1,function_macro_insertion_index);
							macros_function_arglist.insertNSlotsAt(1,function_macro_insertion_index);
							macros_function_expansion.insertNSlotsAt(1,function_macro_insertion_index);
							macros_function_expansion_pre_eval.insertNSlotsAt(1,function_macro_insertion_index);
							macros_function[function_macro_insertion_index] = C_make_string(TokenList[i]->data()+critical_offset,first_token_len);

							macros_function_arglist[function_macro_insertion_index] = new Token<char>;
							macros_function_expansion[function_macro_insertion_index] = new Token<char>;
							macros_function_expansion_pre_eval[function_macro_insertion_index] = new Token<char>(expansion);
							*macros_function_arglist[function_macro_insertion_index] = std::move(arglist);
							*macros_function_expansion[function_macro_insertion_index] = std::move(expansion);
							if (flush_bad_stringize(*macros_function_expansion_pre_eval[function_macro_insertion_index],*macros_function_arglist[function_macro_insertion_index]))
								{
								delete macros_function_expansion_pre_eval[function_macro_insertion_index];
								macros_function_expansion_pre_eval[function_macro_insertion_index] = NULL;
								};
							if (NULL!=macros_function_expansion_pre_eval[function_macro_insertion_index])
								{
								function_macro_concatenate_novars(*macros_function_expansion_pre_eval[function_macro_insertion_index], *macros_function_arglist[function_macro_insertion_index]);
								if ((5>macros_function_arglist[function_macro_insertion_index]->size() || strcmp(macros_function_arglist[function_macro_insertion_index]->end()-(sizeof("...)")-1),"...)")) && C99_VA_ARGS_flinch(*macros_function_expansion_pre_eval[function_macro_insertion_index],0))
									{	//! \test define.C99/Error_concatenate6.hpp, define.C99/Error_concatenate6.h
										//! \test define.C99/Pass_concatenate3.hpp, define.C99/Pass_concatenate3.h
									message_header(*macros_function_expansion_pre_eval[function_macro_insertion_index]);
									INFORM("discarding macro");
									macros_function.DeleteIdx(function_macro_insertion_index);
									macros_function_arglist.DeleteIdx(function_macro_insertion_index);
									macros_function_expansion.DeleteIdx(function_macro_insertion_index);
									macros_function_expansion_pre_eval.DeleteIdx(function_macro_insertion_index);
									TokenList.DeleteIdx(i);
									if (0==i) goto Restart;
									--i;
									continue;
									}
								};
							//! \todo should do locked macros [not __FILE__ or __LINE__] that are not first or last identifiers on speculation
							};
						TokenList.DeleteIdx(i);
						if (0==i) goto Restart;
						--i;
						continue;
						}
					// should use #pragma directives
					// all three STDC pragmas are scoped: they have to be around until fairly late in the compilation sequence
					// we are required to ignore unrecognized pragmas -- no warnings
					// want immediately: #pragma ZCC lock ___
					// * locks macros
					if (PP::PRAGMA==directive_type)
						{
						const size_t critical_offset = valid_directives[directive_type].second+2;
#ifdef BUILD_Z_CPP
						interpret_pragma(TokenList[i]->data() + critical_offset, TokenList[i]->size() - critical_offset, locked_macros);
						TokenList.DeleteIdx(i);
						if (0 == i) goto Restart;
						--i;
						continue;
#else
						switch(const unsigned int pragma_code = interpret_pragma(TokenList[i]->data() + critical_offset, TokenList[i]->size() - critical_offset, locked_macros))
						{
						case RELAY_ZCC_ENABLE_TYPEID + 1:
							TokenList[i]->replace_once(0, TokenList[i]->size(), pragma_relay_keywords[pragma_code - 1].first, pragma_relay_keywords[pragma_code - 1].second);
							continue;
						default:
							TokenList.DeleteIdx(i);
							if (0==i) goto Restart;
							--i;
							continue;
						}
#endif
						}
					}
				}

			if (   PP::IF    ==directive_type
				|| PP::IFDEF ==directive_type
				|| PP::IFNDEF==directive_type)
				{
				if (1 == ++if_depth)
					{
					if_where = i;
					else_where = 0;
					// front-load syntax checking
					// only report errors once
					if (!(TokenList[i]->flags & INVALID_DIRECTIVE_FLAG) && 0==include_where)
						{
						if (PP::IF==directive_type)
							{
							if (!if_elif_syntax_ok(*TokenList[i],macros_object,macros_object_expansion,macros_function,macros_function_arglist,macros_function_expansion_pre_eval,min_types))
								TokenList[i]->flags |= INVALID_DIRECTIVE_FLAG;
							}
						else{
							if (!ifdef_ifndef_syntax_ok(*TokenList[i],macros_object,macros_function))
								TokenList[i]->flags |= INVALID_DIRECTIVE_FLAG;
							else
								assert(PP::IF==UNPACK_DIRECTIVE(TokenList[i]->flags));
							}
						}
					if (0==restart_full_scan)
						{
						restart_full_scan = i+1;
						//! \todo do it all, have complete information
						};
					//! \todo front-load syntax checking, etc. when safe [0==include_where && if_where+1==restart_full_scan]
					//! can detect gross-invalidity anyway, but anything requiring authoritiatively knowing macro (un)defined-ness won't go through
					}
				}
			else if (   PP::ELSE ==directive_type
					 || PP::ELIF ==directive_type
					 || PP::ENDIF==directive_type)
				{
				assert(0<if_depth);
				if (1==if_depth)
					{
					if (PP::ELIF!=directive_type)
						//! test Error_else_with_tokens.hpp : #else with tokens
						//! test Error_endif_with_tokens.hpp : #endif with tokens
						truncate_illegal_tokens(*TokenList[i],directive_type,valid_directives[directive_type].second+1);
					else if (	!(TokenList[i]->flags & INVALID_DIRECTIVE_FLAG)
							 && !if_elif_syntax_ok(*TokenList[i],macros_object,macros_object_expansion,macros_function,macros_function_arglist,macros_function_expansion_pre_eval,min_types))
						TokenList[i]->flags |= INVALID_DIRECTIVE_FLAG;
					};

				if (PP::ENDIF==directive_type)
					{
					if (0 == --if_depth)
						{	// trigger conditional preprocessing here
						assert(TokenList[if_where]->flags & PREPROCESSING_DIRECTIVE_FLAG);
						assert(0==else_where || (TokenList[else_where-1]->flags & PREPROCESSING_DIRECTIVE_FLAG));
						assert(0==else_where || (if_where<=else_where && else_where<=i));
						if (TokenList[if_where]->flags & INVALID_DIRECTIVE_FLAG)
							{	// already tagged as invalid
							message_header(*TokenList[if_where]);
							INFORM("Ignoring entire block controlled by flawed directive");
							TokenList.DeleteNSlotsAt(i-if_where+1,if_where);
							i = if_where;
							if (0==i) goto Restart;
							if (restart_full_scan>=i+1) restart_full_scan = 0;	// very possible
							if (include_where>=i+1) include_where = 0;			// failsafing
							--i;
							continue;
							};

						// inject reductions here
						const unsigned int else_directive = (0==else_where) ? PP_INVALID : UNPACK_DIRECTIVE(TokenList[else_where-1]->flags);
						assert(0==else_where || PP::ELSE==else_directive || PP::ELIF==else_directive);
						if (!strcmp(TokenList[if_where]->data(),"#if 0"))
							{
							if (0==else_where)
								{
								TokenList.DeleteNSlotsAt(i-if_where+1,if_where);
								i = if_where;
								if (0==i) goto Restart;
								if (restart_full_scan>=i+1) restart_full_scan = 0;	// very possible
								if (include_where>=i+1) include_where = 0;			// failsafing
								--i;
								continue;
								};
							if (PP::ELSE==else_directive)
								{
								TokenList.DeleteIdx(i);
//								TokenList.DeleteNSlotsAt((else_where-1)-if_where+1,if_where);
								TokenList.DeleteNSlotsAt(else_where-if_where,if_where);
								i = if_where;
								if (0==i) goto Restart;
								if (restart_full_scan>=i+1) restart_full_scan = 0;	// very possible
								if (include_where>=i+1) include_where = 0;			// failsafing
								--i;
								continue;
								};
							if (TokenList[else_where-1]->flags & INVALID_DIRECTIVE_FLAG)
								{	//! \test Error8.hpp : #elif no control expression, critical
								message_header(*TokenList[else_where-1]);
								INFORM("Ignoring all lines from invalid #elif to matching #endif");
								TokenList.DeleteNSlotsAt(i-if_where+1,if_where);
								i = if_where;
								if (0==i) goto Restart;
								if (restart_full_scan>=i+1) restart_full_scan = 0;	// very possible
								if (include_where>=i+1) include_where = 0;			// failsafing
								--i;
								continue;
								};
							//! \todo do full reduction here to make errors read #elif rather than #if
							// recurse it
							TokenList[else_where-1]->c_array()[2] = '#';
							TokenList[else_where-1]->ltrim(2);
							PACK_DIRECTIVE(TokenList[else_where-1]->flags,PP::IF);
//							TokenList.DeleteNSlotsAt((else_where-1)-if_where+1,if_where);
							TokenList.DeleteNSlotsAt(else_where-if_where+2,if_where);
							i = if_where;
							if (0==i) goto Restart;
							if (restart_full_scan>=i+1) restart_full_scan = 0;	// very possible
							if (include_where>=i+1) include_where = 0;			// failsafing
							--i;
							continue;
							};
						if (!strcmp(TokenList[if_where]->data(),"#if 1"))
							{
							if (0==else_where)
								{
								TokenList.DeleteIdx(i);
								TokenList.DeleteIdx(if_where);
								i = if_where;
								if (0==i) goto Restart;
								if (restart_full_scan>=i+1) restart_full_scan = 0;	// very possible
								if (include_where>=i+1) include_where = 0;			// failsafing
								--i;
								continue;
								};
							if (PP::ELIF==else_directive && (TokenList[else_where-1]->flags & INVALID_DIRECTIVE_FLAG))
								{	//! \test Error9.hpp : #elif no control expression, non-critical
								message_header(*TokenList[else_where-1]);
								INFORM("Continuing as expression for invalid #elif not needed.");
								};
//							TokenList.DeleteNSlotsAt(i-(else_where-1)+1,else_where-1);
							TokenList.DeleteNSlotsAt(i-else_where+2,else_where-1);
							TokenList.DeleteIdx(if_where);
							i = if_where;
							if (0==i) goto Restart;
							if (restart_full_scan>=i+1) restart_full_scan = 0;	// very possible
							if (include_where>=i+1) include_where = 0;			// failsafing
							--i;
							continue;
							};
						}
					}
				else if (1==if_depth && 0==else_where)
					else_where = i+1;
				}
			}
		// non-directive; lex, and check for macros and _Pragma operators
		// remember to convert whitespace to single-space tokens, and flush those later
		else if (0==include_where && 0==restart_full_scan)
			{
			if (!tokenize_line(TokenList,i))
				{
				if (0==i) goto Restart;
				--i;
				continue;
				};
			if (C_TESTFLAG_PP_OP_PUNC & TokenList[i]->flags)
				{	// check for categorically illegal tokens
				const signed int old_pp_code = C_PP_DECODE(TokenList[i]->flags);
				const signed int pp_code = (old_pp_code) ? old_pp_code : lang.pp_support->EncodePPOpPunc(TokenList[i]->data(),TokenList[i]->size());
				assert(0<pp_code);
				if (C_DISALLOW_POSTPROCESSED_SOURCE & lang.pp_support->GetPPOpPuncFlags(pp_code))
					{	//! \todo need test cases
						// actually, this might need to be language-sensitive (e.g., Perl)
					message_header(*TokenList[i]);
					INC_INFORM(ERR_STR);
					INC_INFORM("Forbidden token '");
					INC_INFORM(TokenList[i]->data(),TokenList[i]->size());
					INFORM("' in postprocessed source.  Discarding.");
					zcc_errors.inc_error();
					TokenList.DeleteIdx(i);
					if (0==i) goto Restart;
					--i;
					continue;
					}
				C_PP_ENCODE(TokenList[i]->flags,pp_code);
				}
			else if (C_TESTFLAG_IDENTIFIER==TokenList[i]->flags)
				{
				if (!strcmp(TokenList[i]->data(),"_Pragma"))
					{	// could be pragma operator; syntax _Pragma ( C-string )
					while(TokenList.size()>i+1 && !tokenize_line(TokenList,i+1));
					if (        TokenList.size()<=i+1
						||   1!=TokenList[i+1]->size()
						|| '('!=TokenList[i+1]->front())
						{	//! \test cpp/Pragma.C99/Error_op1.hpp, cpp/Pragma.C99/Error_op1.h
							//! \test cpp/Pragma.C99/Error_op2.hpp, cpp/Pragma.C99/Error_op2.h
						message_header2(*TokenList[i],TokenList[i]->logical_line.second);
						INC_INFORM(ERR_STR);
						INFORM("Invalid _Pragma operator.  Discarding. (C99 6.10.9p1/C++0x 16.9)");
						zcc_errors.inc_error();
						TokenList.DeleteIdx(i);
						if (0==i) goto Restart;
						--i;
						continue;
						};
					while(TokenList.size()>i+2 && !tokenize_line(TokenList,i+2));
					if (   TokenList.size()<=i+2
						|| C_TESTFLAG_STRING_LITERAL!=TokenList[i+2]->flags)
						{	//! \test cpp/Pragma.C99/Error_op3.hpp, cpp/Pragma.C99/Error_op3.h
							//! \test cpp/Pragma.C99/Error_op4.hpp, cpp/Pragma.C99/Error_op4.h
						message_header2(*TokenList[i],TokenList[i]->logical_line.second);
						INC_INFORM(ERR_STR);
						INFORM("Invalid _Pragma operator.  Discarding. (C99 6.10.9p1/C++0x 16.9)");
						zcc_errors.inc_error();
						TokenList.DeleteNSlotsAt(2,i);
						if (0==i) goto Restart;
						--i;
						continue;
						};
					while(TokenList.size()>i+3 && !tokenize_line(TokenList,i+3));
					if (        TokenList.size()<=i+3
						||   1!=TokenList[i+3]->size()
						|| ')'!=TokenList[i+3]->front())
						{	//! \test cpp/Pragma.C99/Error_op5.hpp, cpp/Pragma.C99/Error_op5.h
							//! \test cpp/Pragma.C99/Error_op6.hpp, cpp/Pragma.C99/Error_op6.h
						message_header2(*TokenList[i],TokenList[i]->logical_line.second);
						INC_INFORM(ERR_STR);
						INFORM("Invalid _Pragma operator.  Discarding. (C99 6.10.9p1/C++0x 16.9)");
						zcc_errors.inc_error();
						TokenList.DeleteNSlotsAt(3,i);
						if (0==i) goto Restart;
						--i;
						continue;
						};
					if ('L'==TokenList[i+2]->front())
						TokenList[i+2]->ltrim(1);
					if (2<TokenList[i+2]->size())
						{	//! \test Pass_pragma_STDC.hpp
						autovalarray_ptr_throws<char> pragma_string(lang.UnescapeStringLength(TokenList[i+2]->data()+1,TokenList[i+2]->size()-2));
						lang.UnescapeString(pragma_string.c_array(),TokenList[i+2]->data()+1,TokenList[i+2]->size()-2);
#ifdef BUILD_Z_CPP
						interpret_pragma(pragma_string.data(), pragma_string.size(), locked_macros);
#else
						switch(const unsigned int pragma_code = interpret_pragma(pragma_string.data(), pragma_string.size(), locked_macros))
						{
						case RELAY_ZCC_ENABLE_TYPEID+1:
							TokenList[i]->replace_once(0,TokenList[i]->size(),pragma_relay_keywords[pragma_code-1].first,pragma_relay_keywords[pragma_code-1].second);
							TokenList.DeleteNSlotsAt(3,i+1);
							continue;
						}
#endif
						};
					TokenList.DeleteNSlotsAt(4,i);
					if (0==i) goto Restart;
					--i;
					continue;
					}

				const errr object_macro_index = binary_find(TokenList[i]->data(),TokenList[i]->size(),macros_object);
				const errr function_macro_index = binary_find(TokenList[i]->data(),TokenList[i]->size(),macros_function);
				assert(0>object_macro_index || 0>function_macro_index);
				if (0<=object_macro_index)
					{	// object-like macro
					if (!macros_object_expansion_pre_eval[object_macro_index])
						{	// expands to nothing
							//! \test cpp/default/Preprocess_empty_macros.hpp, cpp/default/Preprocess_empty_macros.h
						TokenList.DeleteIdx(i);
						if (0==i) goto Restart;
						--i;
						continue;
						}
					assert(!macros_object_expansion_pre_eval[object_macro_index]->empty());
					{	//! \test cpp/default/Preprocess_*.h/hpp
					size_t discard = 0;
					dynamic_macro_replace_once(*TokenList[i],discard,TokenList[i]->size(),macros_object,macros_object_expansion_pre_eval,macros_function,macros_function_arglist,macros_function_expansion_pre_eval,NULL);
					}
					size_t actual_tokens = tokenize_line(TokenList,i);
					assert(0<actual_tokens);
					i += actual_tokens-1;
					}
				else if (0<=function_macro_index)
					{	// could be function-like macro
					if (	TokenList.size()>i+1 && TokenList[i]->logical_line.first==TokenList[i+1]->logical_line.first
						&& 	TokenList[i]->logical_line.second+TokenList[i]->size()==TokenList[i+1]->logical_line.second
						&&	'('==TokenList[i+1]->front())
						{
						size_t paren_depth = 1;
						size_t comma_count = 0;
						size_t j = i+1;
						do	{
							if (TokenList.size()<=j+1)
								{	//! \test cpp/Error_macro_arglist4.hpp
									// error out, incomplete function-like macro
								message_header2(*TokenList[i],TokenList[i]->logical_line.second);
								INC_INFORM(ERR_STR);
								INC_INFORM("macro ");
								INC_INFORM(TokenList[i]->data(),TokenList[i]->size());
								INFORM(" did not close its argument list in time. (C99 6.10p1/C++98 16.1p1)");
								zcc_errors.inc_error();
								i = j;
								break;
								}
							if (TokenList[j]->logical_line.first<TokenList[j+1]->logical_line.first)
								{	// line advance; check for pp-directives (undefined behavior), then tokenize
								if (line_is_preprocessing_directive(*TokenList[j+1]))
									{	//! \test cpp/Error_macro_arglist7.hpp
										// error out, undefined behavior
									message_header2(*TokenList[i],TokenList[i]->logical_line.second);
									INC_INFORM(ERR_STR);
									INFORM("macro invocation contains preprocessing directive.  Defining undefined behavior as ignoring macro invocation. (C99 6.10.3p11/C++98 16.3p10)");
									zcc_errors.inc_error();
									i = j;
									break;
									}
								if (!tokenize_line(TokenList,j+1)) continue;
								}
							if (1==TokenList[++j]->size())
								switch(TokenList[j]->front())
								{
								case '(':
									++paren_depth;
									break;
								case ',':
									++comma_count;
									break;
								case ')':
									--paren_depth;
									//	break;
								};
							}
						while(0<paren_depth);
						if (0==paren_depth)
							{
							assert(macros_function_arglist[function_macro_index]);
							assert('('==macros_function_arglist[function_macro_index]->front());
							assert(')'==macros_function_arglist[function_macro_index]->back());
							const size_t formal_arg_span = macros_function_arglist[function_macro_index]->size();
							const size_t formal_arg_count = (2<formal_arg_span) ? std::count(macros_function_arglist[function_macro_index]->begin(),macros_function_arglist[function_macro_index]->end(),',')+1 : 0;
							const bool formal_varadic = 5<=formal_arg_span && !strncmp(macros_function_arglist[function_macro_index]->data()+(formal_arg_span-4),"...",sizeof("...")-1);
							const size_t arg_count = (i+2==j) ? 0 : comma_count+1;
							if (arg_count<formal_arg_count || (arg_count>formal_arg_count && !formal_varadic))
								{	//! \test cpp/Error_macro_arglist5.hpp
									//! \test cpp/Error_macro_arglist6.hpp
								message_header2(*TokenList[i],TokenList[i]->logical_line.second);
								INC_INFORM(ERR_STR);
								INC_INFORM("macro ");
								INC_INFORM(TokenList[i]->data(),TokenList[i]->size());
								INC_INFORM(" had ");
								INC_INFORM(arg_count);
								INC_INFORM(" argument");
								INC_INFORM((1==arg_count) ? "" : "s");
								INC_INFORM(", needed ");
								if (formal_varadic) INC_INFORM("at least ");
								INC_INFORM(formal_arg_count);
								INFORM(". (C99 6.10p1/C++0x 16.1p1)");
								zcc_errors.inc_error();
								i = j;
								continue;
								}
							if (!macros_function_expansion_pre_eval[function_macro_index])
								{	// expands to nothing
									//! \test cpp/default/Preprocess_empty_macros.hpp, cpp/default/Preprocess_empty_macros.h
								TokenList.DeleteNSlotsAt(j-i+1,i);
								if (0==i) goto Restart;
								--i;
								continue;
								}
							assert(!macros_function_expansion_pre_eval[function_macro_index]->empty());
							{	//! \test default/Preprocess_*.h/hpp 
							autoval_ptr<Token<char> > Tmp;
							Tmp = new Token<char>(*macros_function_expansion_pre_eval[function_macro_index]);
							Tmp->logical_line = TokenList[i]->logical_line;
							if (!nonrecursive_macro_replacement_list(Tmp->data()))
								{	// XXX trashes line information to reuse intrapreprocessing stuff
								size_t discard = i;
								autoval_ptr<Token<char> > Tmp2;
								Tmp2 = new Token<char>(*TokenList[i]);
								while(++discard <= j) Tmp2->append(TokenList[discard]->data());
								discard = 0;
								dynamic_macro_replace_once(*Tmp2,discard,TokenList[i]->size(),macros_object,macros_object_expansion_pre_eval,macros_function,macros_function_arglist,macros_function_expansion_pre_eval,NULL);
								Tmp2.MoveInto(Tmp);
								}
							TokenList.DeleteNSlotsAt(j-i,i+1);
							delete TokenList[i];
							TokenList[i] = Tmp.release();
							}
							size_t actual_tokens = tokenize_line(TokenList,i);
							assert(0<actual_tokens);
							i += actual_tokens-1;
							};
						}
					}
				else	// replace predefined macros, if they are here
					//! \test cpp/default/Preprocess_STDC_defines.hpp, cpp/default/Preprocess_STDC_defines.h
					predefined_macro_replacement(*TokenList[i],0);
				}
			}
		}
	while(++i<TokenList.size());

	if (0<if_depth)
		{	//! \test Error_naked_if.hpp
		message_header(*TokenList[if_where]);
		INC_INFORM(ERR_STR);
		INC_INFORM(if_depth);
		INC_INFORM(" #if/#ifdef/#ifndef");
		if (1<if_depth) INC_INFORM("'s");
		INFORM(" without matching #endif directives (first one here).  (C99 6.10p1/C++98 16.1p1)");
		zcc_errors.inc_error();
		}
	assert(0==include_where || 0==restart_full_scan || include_where<restart_full_scan);

	if (0<include_where)
		{	// Need to find the file....
		--include_where;
		assert(PP::INCLUDE == UNPACK_DIRECTIVE(TokenList[include_where]->flags));
		if (TokenList[include_where]->size()<=(sizeof("#include <>")-1))
			{	//! \test Error_include_tiny1.hpp
				//! \test Error_include_tiny2.hpp
			message_header(*TokenList[include_where]);
			INC_INFORM(ERR_STR);
			INFORM("#include is too small to contain a valid filename; discarding and continuing (C99 6.10.2p2,3/C++98 16.2p2,3)");
			TokenList.DeleteIdx(include_where);
			zcc_errors.inc_error();
			i = include_where;
			if (i<TokenList.size()) goto RestartAfterInclude;
			return;
			}
		// already preprocessed, fortunately
		bool local_include = false;
		size_t filename_len = TokenList[include_where]->size()-(sizeof("#include <>")-1);
		switch(TokenList[include_where]->data()[sizeof("#include ")-1])
		{
		case '"':	// local-include
			if ('"'!=TokenList[include_where]->back())
				{	//! \test Error_include_unterminated2.hpp
				message_header(*TokenList[include_where]);
				INC_INFORM(ERR_STR);
				INFORM("#include \"... does not terminate properly; discarding and continuing (C99 6.10.2p3/C++98 16.2p3)");
				TokenList.DeleteIdx(include_where);
				zcc_errors.inc_error();
				i = include_where;
				if (i<TokenList.size()) goto RestartAfterInclude;
				return;
				};
			local_include = true;
			break;
		case '<':	// system-include
			if ('>'!=TokenList[include_where]->back())
				{	//! \test Error_include_unterminated1.hpp
				message_header(*TokenList[include_where]);
				INC_INFORM(ERR_STR);
				INFORM("#include <... does not terminate properly; discarding and continuing (C99 6.10.2p2/C++98 16.2p2)");
				TokenList.DeleteIdx(include_where);
				zcc_errors.inc_error();
				i = include_where;
				if (i<TokenList.size()) goto RestartAfterInclude;
				return;
				};
			break;
		default:	// neither
			message_header(*TokenList[include_where]);
			INC_INFORM(ERR_STR);
			INFORM("#include has bad format; discarding and continuing (C99 6.10.2p2,3/C++98 16.2p2,3)");
			TokenList.DeleteIdx(include_where);
			zcc_errors.inc_error();
			i = include_where;
			if (i<TokenList.size()) goto RestartAfterInclude;
			return;
		};
		// iterate through search path until something found matching
		// need to map following
		// * filename given from #include
		// * full path that actually finds the file, or positive failure to find (AtomicString)
		// * cached content (probably separate indexing on full-path), if not include-guarded
		// * whether include-guarded #define; if so, what the include-guard macro is...
		// * whether the subject of a pragma-once
		// We need a de-facto stack for the "most recent include dirctory" to deal with local includes
#define CPP_INCLUDE_NOT_FOUND 1U

		autovalarray_ptr<Token<char>* > IncludeTokenList;
#ifdef NO_LEGACY_FIELDS
		std::string look_for(TokenList[include_where]->data() + sizeof("#include <") - 1, filename_len);
#else
		zaimoni::flyweight<const char> look_for(C_string_to_flyweight(TokenList[include_where]->data()+sizeof("#include <")-1,filename_len));
#endif
		assert(filename_len==strlen(look_for.data()));
		if (local_include)
			{	// #include "..." prohibits interior "
			if (strchr(look_for.data(),'"'))
				{	//! \test Error_include_multiterminated2.hpp
				message_header(*TokenList[include_where]);
				INC_INFORM(ERR_STR);
				INFORM("#include \"...\" contains \"; discarding and continuing (C99 6.10.2p3/C++98 16.2p3)");
				TokenList.DeleteIdx(include_where);
				zcc_errors.inc_error();
				i = include_where;
				if (i<TokenList.size()) goto RestartAfterInclude;
				return;
				}
			}
		else if (strchr(look_for.data(),'>')) // #include <...> prohibits interior >
			{	//! \test Error_include_multiterminated1.hpp
			message_header(*TokenList[include_where]);
			INC_INFORM(ERR_STR);
			INFORM("#include <...> contains >; discarding and continuing (C99 6.10.2p2/C++98 16.2p2)");
			TokenList.DeleteIdx(include_where);
			zcc_errors.inc_error();
			i = include_where;
			if (i<TokenList.size()) goto RestartAfterInclude;
			return;
			};

		errr have_file_index = binary_find(look_for.data(), filename_len, include_file_index);
		// system includes use their handle for information.
		//! \todo rewrite this to support the #include_next extension?
		if (0<=have_file_index)
			{	// already have this one....
			if (CPP_INCLUDE_NOT_FOUND==include_file_index[have_file_index].third)
				{	// don't error again....
				TokenList.DeleteIdx(include_where);
				i = include_where;
				if (i<TokenList.size()) goto RestartAfterInclude;
				return;
				}
			assert(NULL!=include_file_index[have_file_index].second);
			const errr cache_index = binary_find(include_file_index[have_file_index].second, strlen(include_file_index[have_file_index].second),include_file_cache);
			assert(0<=cache_index);
			IncludeTokenList = *include_file_cache[cache_index].second;
		 	}
		else{
			char buf[FILENAME_MAX];
			const char* main_index_name = NULL;
			// note: local_include needs to know where to start...
			bool found_file = local_include && find_local_include(look_for.data(), buf);
			bool hardcoded_header = false;
			if (found_file)
				{	// filepath known; local includes use the calculated path for information
				main_index_name = register_string(buf);
				have_file_index = binary_find(main_index_name, strlen(main_index_name), include_file_index);
				if (0<=have_file_index)
					{
					assert(CPP_INCLUDE_NOT_FOUND!=include_file_index[have_file_index].third);
					const errr cache_index = binary_find(main_index_name,strlen(main_index_name),include_file_cache);
					assert(0<=cache_index);
					IncludeTokenList = *include_file_cache[cache_index].second;
					}
				else{	// filepath known; inhale and set up cache
					if (!load_sourcefile(IncludeTokenList,buf,lang)) throw std::bad_alloc();
					// backfit SourceFile... to look_for; parent_dir to parent directory of file in buf
					if (!IncludeTokenList.empty())
						{
#ifdef NO_LEGACY_FIELDS
						auto staging = canonical_cache<std::filesystem::path>::get().track(std::filesystem::path(buf));
						size_t j = IncludeTokenList.size();
						do IncludeTokenList[--j]->src = staging;
						while (0 < j);
#else
						char parent_path[FILENAME_MAX];
						z_dirname(parent_path,buf);
						zaimoni::flyweight<const char> parent_dir(C_string_to_flyweight(parent_path,strlen(parent_path)));
						size_t j = IncludeTokenList.size();
						do	{
							IncludeTokenList[--j]->src_filename = look_for;
							IncludeTokenList[j]->parent_dir = parent_dir;
							}
						while(0<j);
#endif
						};
					// set up include_file_index, include_file_cache
					const size_t include_file_index_target = BINARY_SEARCH_DECODE_INSERTION_POINT(have_file_index);
					const errr cache_index = binary_find(main_index_name,strlen(main_index_name),include_file_cache);
					const size_t include_file_cache_target = BINARY_SEARCH_DECODE_INSERTION_POINT(cache_index);
					include_file_index.insertNSlotsAt(1,include_file_index_target);
					include_file_cache.insertNSlotsAt(1,include_file_cache_target);
					include_file_cache[include_file_cache_target].second = new autovalarray_ptr<Token<char>* >(IncludeTokenList);
					include_file_cache[include_file_cache_target].first = main_index_name;
					include_file_index[include_file_index_target].first = main_index_name;
					include_file_index[include_file_index_target].second = main_index_name;
					include_file_index[include_file_index_target].third = 0;
					}
				}
			else{	// we're having to process it as a system include.
				errr tmp = 0;
				// C99 7.1.2p4 and C++98 17.3.3.1.1p2 both prohibit having any keywords defined as macros when including a library header.
				// C++0x 17.4.3.2.2 goes further and simply prohibits defining any keywords as macros.  Oops.
				// C0x appears to be conserving C99 exactly.
				// We undefine the offending macros as well as erroring, mainly to prevent duplicate error messages
				// C99 doesn't care about such defines *after* the header; C++98 does.
				//! \test cpp/default/keywords/Error_*.h
				if (Lang::C==lang_code && 0<lang.pp_support->LengthOfSystemHeader(look_for.data()))
					C99_reject_keyword_macros(TokenList,include_where,look_for.data(),lang,macros_object,macros_object_expansion,macros_object_expansion_pre_eval,macros_function,macros_function_arglist,macros_function_expansion,macros_function_expansion_pre_eval);

				const unsigned int hardcoded_header_idx = detect_hardcoded_system_header(look_for.data(),lang_code);
				switch(hardcoded_header_idx)
				{
#ifndef NDEBUG
				default: FATAL("hardcoded_header_idx out of range");
#endif
				case 3:	// stdint.h/cstdint
					hardcoded_header = true;
					C99_reject_keyword_macros(TokenList,include_where,look_for.data(),lang,macros_object,macros_object_expansion,macros_object_expansion_pre_eval,macros_function,macros_function_arglist,macros_function_expansion,macros_function_expansion_pre_eval);
					if (0>binary_find("__STDINT_H__",sizeof("__STDINT_H__")-1,macros_object))	
						create_stdint_header(IncludeTokenList,look_for.data());	// not included yet
					break;
				case 2:	// stddef.h/cstddef
					hardcoded_header = true;
					C99_reject_keyword_macros(TokenList,include_where,look_for.data(),lang,macros_object,macros_object_expansion,macros_object_expansion_pre_eval,macros_function,macros_function_arglist,macros_function_expansion,macros_function_expansion_pre_eval);
					if (0>binary_find("__STDDEF_H__",sizeof("__STDDEF_H__")-1,macros_object))	
						create_stddef_header(IncludeTokenList,look_for.data());	// not included yet
					break;
				case 1:	// limits.h/climits
					hardcoded_header = true;
					C99_reject_keyword_macros(TokenList,include_where,look_for.data(),lang,macros_object,macros_object_expansion,macros_object_expansion_pre_eval,macros_function,macros_function_arglist,macros_function_expansion,macros_function_expansion_pre_eval);
					if (0>binary_find("__LIMITS_H__",sizeof("__LIMITS_H__")-1,macros_object))	
						create_limits_header(IncludeTokenList,look_for.data());	// not included yet
					break;
				case 0:;
				}

				found_file = !hardcoded_header && find_system_include(look_for.data(), buf);
				if (found_file)
					{	// filepath known; inhale and set up cache
					if (!load_sourcefile(IncludeTokenList,buf,lang)) throw std::bad_alloc();
					// backfit SourceFile... to look_for; parent_dir to parent directory of file in buf
					if (!IncludeTokenList.empty())
						{
#ifdef NO_LEGACY_FIELDS
						auto staging = canonical_cache<std::filesystem::path>::get().track(std::filesystem::path(buf));
						size_t j = IncludeTokenList.size();
						do IncludeTokenList[--j]->src = staging;
						while (0 < j);
#else
						auto target = std::filesystem::path(buf);
						auto par = target.parent_path();
						auto tmp2 = ((par == origin_dir) ? par : origin_dir).generic_string();
						zaimoni::flyweight<const char> parent_dir(C_string_to_flyweight(tmp2.c_str(), tmp2.size()));
						size_t j = IncludeTokenList.size();
						do {
							IncludeTokenList[--j]->src_filename = look_for;
							IncludeTokenList[j]->parent_dir = parent_dir;
						} while (0 < j);
#endif
						};
					main_index_name = register_string(buf);
					// set up include_file_index, include_file_cache
					tmp = binary_find(look_for.data(),filename_len,include_file_index);
					const size_t include_file_index_target = BINARY_SEARCH_DECODE_INSERTION_POINT(tmp);
					tmp = binary_find(main_index_name,strlen(main_index_name),include_file_cache);
					const size_t include_file_cache_target = BINARY_SEARCH_DECODE_INSERTION_POINT(tmp);
					include_file_index.insertNSlotsAt(1,include_file_index_target);
					include_file_cache.insertNSlotsAt(1,include_file_cache_target);
					include_file_cache[include_file_cache_target].second = new autovalarray_ptr<Token<char>* >(IncludeTokenList);
					include_file_cache[include_file_cache_target].first = main_index_name;
					include_file_index[include_file_index_target].first = look_for.data();
					include_file_index[include_file_index_target].second = main_index_name;
					include_file_index[include_file_index_target].third = 0;
					}
				else if (!hardcoded_header)
					{	// not there at all...
					// set up include_file_index
					tmp = binary_find(look_for.data(),filename_len,include_file_index);
					const size_t include_file_index_target = BINARY_SEARCH_DECODE_INSERTION_POINT(tmp);
					include_file_index.insertNSlotsAt(1,include_file_index_target);
					include_file_index[include_file_index_target].first = look_for.data();
					include_file_index[include_file_index_target].second = NULL;
					include_file_index[include_file_index_target].third = CPP_INCLUDE_NOT_FOUND;

					message_header(*TokenList[include_where]);
					INC_INFORM(ERR_STR);
					INFORM("#include'd file not found; discarding and continuing");
					TokenList.DeleteIdx(include_where);
					zcc_errors.inc_error();
					i = include_where;
					if (i<TokenList.size()) goto RestartAfterInclude;
					return;
					}
				}
			}
#undef CPP_INCLUDE_NOT_FOUND
		//! also tested in a number of data transform tests
		//! \test Pass_include_local.hpp
		if (!IncludeTokenList.empty())
			{	// not sure whether C99 5.1.1.2 4 requires preprocessing the whole header before pasting, but it permits it
				// this implies:
				// * conditional-compilation directives must be balanced for each file
				// * #line directives never escape files anyway
			++include_level;
			_preprocess(IncludeTokenList, locked_macros, macros_object, macros_object_expansion, macros_object_expansion_pre_eval, macros_function, macros_function_arglist, macros_function_expansion, macros_function_expansion_pre_eval, include_file_index, include_file_cache, min_types);
			--include_level;
			if (!IncludeTokenList.empty())
				{
				size_t j = IncludeTokenList.size();
				TokenList.insertNSlotsAt(j,include_where+1);
				memmove(TokenList.c_array()+include_where+1,IncludeTokenList.data(),j*sizeof(Token<char*>*));
#ifdef ZAIMONI_NULL_REALLY_IS_ZERO
				memset(IncludeTokenList.c_array(),0,j*sizeof(Token<char*>*));
#else
				std::fill(IncludeTokenList.begin(),IncludeTokenList.end(),NULL)
#endif
				}
			}

		// XXX fallthrough hack XXX
		TokenList.DeleteIdx(include_where);
		i = include_where;
		if (i<TokenList.size()) goto RestartAfterInclude;
//		return;
		}
}

bool
CPreprocessor::raw_system_include(const char* const look_for, autovalarray_ptr<Token<char>* >& IncludeTokenList) const
{
	char buf[FILENAME_MAX];
	// raw system include has minimal macro context, so don't worry about legality check

	switch(detect_hardcoded_system_header(look_for,lang_code))
	{
#ifndef NDEBUG
	default: FATAL("detect_hardcoded_system_header() return value out of 0..3 range");
#endif
	case 1:	// header is limits.h/climits
		create_limits_header(IncludeTokenList,look_for);	// not included yet
		return true;
	case 2:	// header is stddef.h/cstddef
		create_stddef_header(IncludeTokenList,look_for);	// not included yet
		return true;
	case 3:	// header is stdint.h/cstdint
		create_stdint_header(IncludeTokenList,look_for);	// not included yet
		return true;
	case 0:;
	}

	if (find_system_include(look_for, buf))
		return load_raw_sourcefile(IncludeTokenList,buf);
	return false;
}

unsigned int
CPreprocessor::interpret_pragma(const char* const x, size_t x_len, autovalarray_ptr<char*>& locked_macros)
{
	autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> > pretokenized;
	lang.line_lex(x, x_len, pretokenized);

	const errr valid_pragma_class = linear_find_lencached(x+pretokenized[0].first, pretokenized[0].second, accept_pragma_leading_tokens, STATIC_SIZE(accept_pragma_leading_tokens));
	if (PRAGMA_LEADING_ZCC==valid_pragma_class)
		{
		if (1<pretokenized.size())
			{
#ifdef BUILD_ZCC
			static_assert(RELAY_ZCC_ENABLE_TYPEID==STATIC_SIZE(pragma_STDC_on_off_switch)*STATIC_SIZE(pragma_STDC_keywords));
#endif
			const errr ZCC_pragma =  linear_find_lencached(x+pretokenized[1].first, pretokenized[1].second, pragma_ZCC_keywords, STATIC_SIZE(pragma_ZCC_keywords));
			switch(ZCC_pragma)
			{
#ifdef BUILD_ZCC
			// #pragma ZCC enable_typeid gets rewritten to the 
			// reserved-to-the-implementation keyword 
			// _ZCC_pragma_enable_typeid, which in turn turns off the syntax
			// errors for typeid .  We use this convolution so that we don't
			// instantly break other compilers inadvertently using our 
			// #include <typeinfo>
			case PRAGMA_ZCC_ENABLE_TYPEID: return STATIC_SIZE(pragma_STDC_on_off_switch)*STATIC_SIZE(pragma_STDC_keywords)+1;
#endif
			case PRAGMA_ZCC_LOCK:
				{	//! \test Error_undef_locked_macro.hpp
				size_t j = pretokenized.size();
				while(2<j)
					{
					if (C_TESTFLAG_IDENTIFIER!=pretokenized[--j].third) continue;
					char* tmp = C_make_string(x+pretokenized[j].first,pretokenized[j].second);
					if (!locked_macros.InsertSlotAt(locked_macros.size(),tmp))
						{
						free(tmp);
						throw std::bad_alloc();
						}
					}
				}
			}
			}
		return 0;
		}
	else if (PRAGMA_LEADING_STDC==valid_pragma_class)
		{
		if (1<pretokenized.size())
			{
			const errr STDC_pragma =  linear_find_lencached(x+pretokenized[1].first, pretokenized[1].second, pragma_STDC_keywords, STATIC_SIZE(pragma_STDC_keywords));
			if (0<=STDC_pragma)
				{	// found something we know
				switch(STDC_pragma)
				{
#ifndef NDEBUG
				default:
					INC_INFORM(ERR_STR);
					INC_INFORM("unhandled STDC pragma ");
					INFORM(pragma_STDC_keywords[STDC_pragma].first);
					zcc_errors.inc_error();
					return 0;
#endif
				case PRAGMA_STDC_FP_CONTRACT:
				case PRAGMA_STDC_FENV_ACCESS:
				case PRAGMA_STDC_CX_LIMITED_RANGE:
					{
					//! \test cpp/Pragma.C99/Error_CX_LIMITED_RANGE1.hpp, cpp/Pragma.C99/Error_CX_LIMITED_RANGE1.h
					//! \test cpp/Pragma.C99/Error_CX_LIMITED_RANGE2.hpp, cpp/Pragma.C99/Error_CX_LIMITED_RANGE2.h
					//! \test cpp/Pragma.C99/Error_FENV_ACCESS1.hpp, cpp/Pragma.C99/Error_FENV_ACCESS1.h
					//! \test cpp/Pragma.C99/Error_FENV_ACCESS2.hpp, cpp/Pragma.C99/Error_FENV_ACCESS2.h
					//! \test cpp/Pragma.C99/Error_FP_CONTRACT1.hpp, cpp/Pragma.C99/Error_FP_CONTRACT1.h
					//! \test cpp/Pragma.C99/Error_FP_CONTRACT2.hpp, cpp/Pragma.C99/Error_FP_CONTRACT2.h
					const errr on_off_switch = (3==pretokenized.size()) ? linear_find_lencached(x+pretokenized[2].first, pretokenized[2].second, pragma_STDC_on_off_switch, STATIC_SIZE(pragma_STDC_on_off_switch)) : -2;
					if (0>on_off_switch)
						{
						INC_INFORM(ERR_STR);
						INC_INFORM("invalid STDC pragma ");
						INFORM(x,x_len);
						zcc_errors.inc_error();
						return 0;
						}
					// valid STDC pragma: relay encoding out
					return STATIC_SIZE(pragma_STDC_on_off_switch)*STDC_pragma+on_off_switch+1;
					};
				}
			}
			}
		return 0;
		}
	else if (PRAGMA_MESSAGE==valid_pragma_class)
		{
		if (	4==pretokenized.size()
			&&	1==pretokenized[1].second && '('==x[pretokenized[1].first]
			&&	1==pretokenized[3].second && ')'==x[pretokenized[3].first]
			&&	C_TESTFLAG_STRING_LITERAL==pretokenized[2].third)
			{	// hmm...do we need to unescape anything...
			const bool wide_str = 'L'==x[pretokenized[2].first];
			if (0<std::count(x+pretokenized[2].first,x+pretokenized[2].first+pretokenized[2].second,'\\'))
				{	// no escapes
				if (wide_str) return 0; //! \todo this should do a proper unescape to UNICODE, then use a wrapper library to push the UNICODE to whatever wide-char support there is

				//! \todo change target, this only handles target CHAR_BIT<=host CHAR_BIT
				const size_t tmp_len = lang.UnescapeStringLength(x+pretokenized[2].first,pretokenized[2].second);
				if (char* tmp = _new_buffer<char>(ZAIMONI_LEN_WITH_NULL(tmp_len)))
					{
					lang.UnescapeString(tmp,x+pretokenized[2].first,pretokenized[2].second);
					INFORM(tmp,tmp_len);
					free(tmp);
					return 0;
					}
				};
			// no escapes, or formatting failed: do something
			if (wide_str)
				INFORM(x+pretokenized[2].first+2,pretokenized[2].second-3);
			else
				INFORM(x+pretokenized[2].first+1,pretokenized[2].second-2);
			}
		return 0;
		}
	return 0;
}

static void _complete_string_character_literal(Token<char>& x,const char delim, const char* const end_error)
{
	if (delim==x.back()) return;
	message_header2(x,x.original_line.second);
	INC_INFORM(ERR_STR);
	INC_INFORM("unterminated");
	if ('L'==x.front()) INC_INFORM(" wide");
	INFORM(end_error);
	zcc_errors.inc_error();
	x.append(delim);
}

static void complete_string_character_literal(Token<char>& x)
{
	if (C_TESTFLAG_STRING_LITERAL==x.flags)
		//! \test Error_unterminated1.hpp
		//! \test Error_unterminated2.hpp
		_complete_string_character_literal(x,'"'," string literal.  Terminating. (C99 6.4.5p1/C++98 2.13.4)");
	else if (C_TESTFLAG_CHAR_LITERAL==x.flags)
		//! \test Error_unterminated3.hpp
		//! \test Error_unterminated4.hpp
		_complete_string_character_literal(x,'\''," character literal.  Terminating. (C99 6.4.4.4p1/C++98 2.13.2)");
}

size_t
CPreprocessor::tokenize_line(autovalarray_ptr<Token<char>* >& TokenList, size_t i) const
{
	assert(TokenList.size()>i);
	if (TokenList[i]->empty()) return TokenList.DeleteIdx(i),0; 

	// Not really (it's a preprocessing directive), but we don't want to damage it here
	if ('#'==TokenList[i]->front()) return 1;

	//! \test Error_naked_VA_ARGS.hpp
	if (!TokenList[i]->flags) C99_VA_ARGS_flinch(*TokenList[i],0);

	autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> > pretokenized;
	lang.line_lex(TokenList[i]->data(),TokenList[i]->size(),pretokenized);
	if (pretokenized.empty()) return TokenList.DeleteIdx(i),0;

	size_t ub = pretokenized.size()-1;
	if (0==ub)
		{
		if (!TokenList[i]->flags) TokenList[i]->flags = pretokenized[ub].third;
		// handle trailing whitespace
		if (ZAIMONI_LEN_WITH_NULL(pretokenized[ub].second)<TokenList[i]->size())
			TokenList[i]->lslice(pretokenized[ub].second);
		complete_string_character_literal(*TokenList[i]);
		}
	else{
		size_t lb = 0;
		autovalarray_ptr_throws<Token<char>* > TokenListAlt(ub+1);
		TokenList.insertNSlotsAt(ub,i+1);
		{
		const Token<char>& tmp = *TokenList[i];
		while(lb<ub)
			{
			if (pretokenized[lb].second<=pretokenized[ub].second)
				{	// first token not longer
				TokenListAlt[lb] = new Token<char>(tmp,pretokenized[lb].first,pretokenized[lb].second,pretokenized[lb].third);
				complete_string_character_literal(*TokenListAlt[lb++]);
				}
			else{	// first token longer
				TokenListAlt[ub] = new Token<char>(tmp,pretokenized[ub].first,pretokenized[ub].second,pretokenized[ub].third);
				complete_string_character_literal(*TokenListAlt[ub--]);
				}
			}
		}
		// strip down TokenList[i] to a largest token
		TokenListAlt[lb] = TokenList[i];
		TokenList[i] = NULL;
		TokenListAlt[lb]->trim(pretokenized[lb].first,TokenListAlt[lb]->size()-(pretokenized[lb].first+pretokenized[lb].second));
		TokenListAlt[lb]->flags = pretokenized[lb].third;
		complete_string_character_literal(*TokenListAlt[lb]);
		memmove(TokenList.c_array()+i,TokenListAlt.data(),TokenListAlt.size()*sizeof(*TokenListAlt.data()));
#ifdef ZAIMONI_NULL_REALLY_IS_ZERO
		memset(TokenListAlt.c_array(),0,TokenListAlt.size()*sizeof(*TokenListAlt.data()));
#else
		std::fill(TokenListAlt.begin(),TokenListAlt.end(),NULL);
#endif
		}
	return pretokenized.size();
}

/*! 
 * finds a local include file
 * 
 * \param src : string describing file to find
 * \param filepath_buf : pointer to character buffer of length FILENAME_MAX
 * 
 * \return true if and only if a filepath was found.
 */
bool CPreprocessor::find_local_include(const char* const src, char* const filepath_buf) const
{
	assert(src && *src);
	assert(filepath_buf);

	std::filesystem::path test(origin_dir.empty() ? src // ahem...hope that we haven't relocated yet...
												  : std::filesystem::path(origin_dir) /= src);
	try {
		auto canon = std::filesystem::canonical(test);
		auto str = canon.generic_string();
		// automatically fail anything that won't fit in FILENAME_MAX
		//! \test Error_huge_path.hpp
		if (FILENAME_MAX <= str.size()) return false;
		strcpy(filepath_buf, str.c_str());
		return true;
	} catch(std::filesystem::filesystem_error& e) {
		//! \todo react to local search path options as well
		//! \test Error_include_nonexistent2.hpp
		return false;
	}
}

/*! 
 * finds a system include file
 * 
 * \param src : string describing file to find
 * \param filepath_buf : pointer to character buffer of length FILENAME_MAX
 * 
 * \return true if and only if a filepath was found.
 */
bool
CPreprocessor::find_system_include(const char* const src, char* const filepath_buf) const
{
	char image_filepath[FILENAME_MAX];
	char test_filepath[FILENAME_MAX];

	assert(src && *src);
	assert(filepath_buf);
	const size_t src_len = strlen(src);
	// automatically fail anything that won't fit in FILENAME_MAX
	//! \test Error_huge_path.hpp
	if (FILENAME_MAX<=src_len) return false;

	// we do casual chroot-jailing of system includes

	// disallow change to parent directory.  *NIX system symlinks mean we can't tolerate them even when balanced properly for Windows.
	if (sizeof("..")-1<=src_len)
		{
		const char* parent_dir_candidate = strstr(src,"..");
		if (src==parent_dir_candidate)
			{	//! \test Error_chroot_jail1.hpp
			if (sizeof("..")-1==src_len) return false;
			//! \test Error_chroot_jail2.hpp
			if ('/'==src[sizeof("..")-1]) return false;
			parent_dir_candidate = strstr(parent_dir_candidate+(sizeof("..")-1),"..");
			}
		while(parent_dir_candidate && src_len>=(parent_dir_candidate-src)+(sizeof("..")-1))
			{	//! \test Error_chroot_jail3.hpp
			if ('/'==parent_dir_candidate[-1]) return false;
			parent_dir_candidate = (src_len==parent_dir_candidate-src+(sizeof("..")-1)) ? NULL : strstr(parent_dir_candidate+(sizeof("..")-1),"..");
			}
		}	

	//!\todo react to system search path options as well

	// check hard-coded search path options
	// \todo more robust multi-language architecture
	//! \test Pass13.hpp
	size_t i = (Lang::C==lang_code) ? START_CPP_ONLY_PATHS : 0;
	do	if (actual_system_include_search[i])
			{
			size_t target_length = strlen(actual_system_include_search[i]);
			assert(FILENAME_MAX>target_length);
			strcpy(test_filepath,actual_system_include_search[i]);
			if (FILENAME_MAX<=target_length+(sizeof(ZAIMONI_PATH_SEP)-1)+src_len) continue;	// safe only because FILENAME_MAX is small
			strcpy(test_filepath+target_length,ZAIMONI_PATH_SEP);
			target_length += (sizeof(ZAIMONI_PATH_SEP)-1);
			strcpy(test_filepath+target_length,src);
#if ZAIMONI_TARGET_FILESYSTEM!=ZAIMONI_TARGET_FILESYSTEM_POSIX
			// path separator not POSIX, ZCC requires POSIX path separator for #include
			//! \todo: actively reject non-POSIX file separators?
			std::replace(test_filepath+target_length,test_filepath+target_length+src_len,'/',ZAIMONI_PATH_SEP_CHAR);
#endif
			const char* const canonical_path = z_realpath(image_filepath,test_filepath);
			if (canonical_path && !access(canonical_path,F_OK))
				{
				strcpy(filepath_buf,canonical_path);
				return true;
				}
			}
	while(STATIC_SIZE(actual_system_include_search) > ++i);
	return false;	//! \test Error_include_nonexistent1.hpp
}

/*! 
 * indicates whether the definedness of the macro is context-free
 * 
 * \param x C string to test for macro-hood
 * \param x_len length of C string
 * 
 * \return 1 if unconditionally defined; -1 if unconditionally undefined; 0 if need to preprocess to find out
 */
int
CPreprocessor::context_free_defined(const char* const x, size_t x_len) const
{
	assert(x && *x);
	assert(0<x_len);
	if (0<=linear_find(x, x_len, macro_identifier_default, macro_identifier_default_count)) return 1;
	// report "magic macros" as defined if the master preprocessor does
	if (sizeof("__has_include")-1==x_len && !strncmp(x,"__has_include",x_len)) return 1;	// CLang says predefined
	if (hard_locked_macro(x,x_len)) return -1;
	return 0;
}

static bool
macro_is_defined(const char* const x, const size_t x_len, const autovalarray_ptr<char*>& macros_object, const autovalarray_ptr<char*>& macros_function)
{
	assert(x && *x);
	assert(0<x_len);
#ifdef NDEBUG
	return 0<=binary_find(x,x_len,macros_object) || 0<=binary_find(x,x_len,macros_function);
#else
	const errr object_macro_index = binary_find(x,x_len,macros_object);
	const errr function_macro_index = binary_find(x,x_len,macros_function);
	assert(0>object_macro_index || 0>function_macro_index);
	return 0<=object_macro_index || 0<=function_macro_index;
#endif
}

bool
CPreprocessor::ifdef_ifndef_syntax_ok(Token<char>& x, const autovalarray_ptr<char*>& macros_object, const autovalarray_ptr<char*>& macros_function)
{
	assert(3<=x.size());
	assert('#'==x.front());
	const unsigned int if_directive = UNPACK_DIRECTIVE(x.flags);
	assert(PP::IFDEF==if_directive || PP::IFNDEF==if_directive);
	assert(!strncmp(x.data()+1,valid_directives[if_directive].first,valid_directives[if_directive].second));
	lex_flags token_flags;

	//! \test ifdef.C99/Error_noarg.hpp
	//! \test ifdef.C99/Error_noarg2.hpp
	if (!strcmp(x.data()+1,valid_directives[if_directive].first))
		{
		message_header(x);
		INC_INFORM(ERR_STR);
		INC_INFORM("#");
		INC_INFORM(valid_directives[if_directive].first);
		INFORM(" has no control expression. (C99 6.10p1/C++98 16p1)");
		zcc_errors.inc_error();
		return false;
		};

	const size_t critical_offset = valid_directives[if_directive].second+2;
	assert(x.size()>critical_offset);
	//! \todo : let this slide as a warning with new --do-what-i-mean option
	C99_VA_ARGS_flinch(x,critical_offset);	// __VA_ARGS__ is known to be undefined, result would be well-defined anyway

	const size_t token_len = lang.UnfilteredNextToken(x.data()+critical_offset,token_flags);
	if (C_TESTFLAG_IDENTIFIER!=token_flags)
		{	//! \test ifdef.C99/Error_nonidentifer.h
			//! \test ifdef.C99/Error_nonidentifer.hpp
			//! \test ifdef.C99/Error_nonidentifer2.h
			//! \test ifdef.C99/Error_nonidentifer2.hpp
		message_header(x);
		INC_INFORM(ERR_STR);
		INC_INFORM("#");
		INC_INFORM(valid_directives[if_directive].first);
		INFORM(" is not applied to an identifier. (C99 6.10p1/C++98 16p1)");
		zcc_errors.inc_error();
		return false;
		};
	if (x.size()-critical_offset>token_len)
		{
		const size_t skip_ws = strspn(x.data()+critical_offset+token_len,lang.WhiteSpace+1);
		if (x.size()-(critical_offset+token_len)>skip_ws)
			{	//! \test ifdef.C99/Error_extra.h
				//! \test ifdef.C99/Error_extra.hpp
				//! \test ifdef.C99/Error_extra2.h
				//! \test ifdef.C99/Error_extra2.hpp
			message_header(x);
			INC_INFORM(ERR_STR);
			INC_INFORM("#");
			INC_INFORM(valid_directives[if_directive].first);
			INFORM(" has illegal tokens following its identifier. (C99 6.10p1/C++98 16p1)");
			zcc_errors.inc_error();
			return false;
			}
		x.rtrim(skip_ws);
		};

	//! \test Pass_macro_STDC.h
	//! \test Pass_macro_STDC.hpp
	switch(context_free_defined(x.data()+critical_offset,token_len))
	{
	case 1:	// found it
		x.replace_once(std::nothrow,0,x.size(),(PP::IFDEF==if_directive) ? "#if 1" : "#if 0");
		PACK_DIRECTIVE(x.flags,PP::IF);
		return true;
	case -1:	// hard-locked, not defined : ergo, undefined
		x.replace_once(std::nothrow,0,x.size(),(PP::IFDEF==if_directive) ? "#if 0" : "#if 1");
		PACK_DIRECTIVE(x.flags,PP::IF);
		return true;
	};

	// exercised heavily by the standard library include tests
	// MingW32 4.3.3 doesn't like factoring out the last two lines
	if (macro_is_defined(x.data()+critical_offset, token_len,macros_object,macros_function))
		{	// found it
		x.replace_once(std::nothrow,0,x.size(),(PP::IFDEF==if_directive) ? "#if 1" : "#if 0");
		PACK_DIRECTIVE(x.flags,PP::IF);
		return true;
		}
	else{	// not found, not a predefined: not defined
		x.replace_once(std::nothrow,0,x.size(),(PP::IFDEF==if_directive) ? "#if 0" : "#if 1");
		PACK_DIRECTIVE(x.flags,PP::IF);
		return true;
		}
}

static POD_pair<size_t,size_t> balanced_character_count(const char* const x, const autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> >& pretokenized,const char l_match,const char r_match)
{
	POD_pair<size_t,size_t> paren_depth = {0,0};
	const autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> >::const_iterator iter_end = pretokenized.end();
	autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> >::const_iterator iter = pretokenized.begin();
	assert(x);
	if (iter!=iter_end)
		do	if (1==iter->second)
				{
				if 		(l_match==x[iter->first]) ++paren_depth.first;
				else if (r_match==x[iter->first]) ++paren_depth.second;
				}
		while(++iter!=iter_end);
	return paren_depth;
}

// This should notice zero errors, thanks to a prior context-free check
static void _construct_matched_pairs(const Token<char>& x, const autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> >& pretokenized, autovalarray_ptr<POD_pair<size_t,size_t> >& pair_stack,const char l_match,const char r_match)
{
	POD_pair<size_t,size_t> depth = balanced_character_count(x.data(),pretokenized,l_match,r_match);	// pre-scan
	DEBUG_STATEMENT(size_t err_count = 0;)
	if (0<depth.first && 0<depth.second)
		{	// reality-check: balanced parentheses
		autovalarray_ptr_throws<size_t> lparen_fixedstack(depth.first);
		autovalarray_ptr_throws<POD_pair<size_t,size_t> > parenpair_fixedstack(depth.first<depth.second ? depth.first : depth.second);
		size_t balanced_paren = 0;
		size_t i = 0;

		depth.first = 0;
		depth.second = 0;
		do	if (1==pretokenized[i].second)
				{
				if 		(l_match==x.data()[pretokenized[i].first])
					{
					if (0<depth.second)
						{
						depth.second = 0;
						DEBUG_STATEMENT(++err_count;)
						}
					lparen_fixedstack[depth.first++] = i;
					}
				else if (r_match==x.data()[pretokenized[i].first])
					{
					if (0<depth.first)
						{
						parenpair_fixedstack[balanced_paren].first = lparen_fixedstack[--depth.first];
						parenpair_fixedstack[balanced_paren++].second = i;
						}
					else
						++depth.second;
					};
				}
		while(pretokenized.size() > ++i);
		if (0==depth.first && 0==depth.second DEBUG_STATEMENT(&& 0==err_count))
			{
			assert(parenpair_fixedstack.size()==balanced_paren);
			parenpair_fixedstack.MoveInto(pair_stack);
			}
		};

	assert(0==depth.first || 0==depth.second);
	DEBUG_STATEMENT(if (0<depth.second) ++err_count;)
	DEBUG_STATEMENT(if (0<depth.first) ++err_count;)
	assert(0==err_count);
}

template<char l_match,char r_match>
inline static void construct_matched_pairs(const Token<char>& x, const autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> >& pretokenized, autovalarray_ptr<POD_pair<size_t,size_t> >& pair_stack)
{
	_construct_matched_pairs(x,pretokenized,pair_stack,l_match,r_match);
}

template<>
void construct_matched_pairs<'[',']'>(const Token<char>& x, const autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> >& pretokenized, autovalarray_ptr<POD_pair<size_t,size_t> >& pair_stack)
{
	POD_pair<size_t,size_t> depth = balanced_character_count(x.data(),pretokenized,'[',']');	// pre-scan
	DEBUG_STATEMENT(size_t err_count = 0;)
	if (0<depth.first && 0<depth.second)
		{	// reality-check: balanced parentheses
		autovalarray_ptr_throws<size_t> lparen_fixedstack(depth.first);
		autovalarray_ptr_throws<POD_pair<size_t,size_t> > parenpair_fixedstack(depth.first<depth.second ? depth.first : depth.second);
		size_t balanced_paren = 0;
		size_t i = 0;

		depth.first = 0;
		depth.second = 0;
		do	if 		(detect_C_left_bracket_op(x.data()+pretokenized[i].first,pretokenized[i].second))
				{
				if (0<depth.second)
					{
					depth.second = 0;
					DEBUG_STATEMENT(++err_count;)
					}
				lparen_fixedstack[depth.first++] = i;
				}
			else if (detect_C_right_bracket_op(x.data()+pretokenized[i].first,pretokenized[i].second))
				{
				if (0<depth.first)
					{
					parenpair_fixedstack[balanced_paren].first = lparen_fixedstack[--depth.first];
					parenpair_fixedstack[balanced_paren++].second = i;
					}
				else
					++depth.second;
				}
		while(pretokenized.size() > ++i);
		if (0==depth.first && 0==depth.second DEBUG_STATEMENT(&& 0==err_count))
			{
			assert(parenpair_fixedstack.size()==balanced_paren);
			parenpair_fixedstack.MoveInto(pair_stack);
			}
		};

	assert(0==depth.first || 0==depth.second);
	DEBUG_STATEMENT(if (0<depth.second) ++err_count;)
	DEBUG_STATEMENT(if (0<depth.first) ++err_count;)
	assert(0==err_count);
}

static void pairstack_clean(const POD_pair<size_t,size_t>& target, autovalarray_ptr<POD_pair<size_t,size_t> >& pair_stack)
{
	autovalarray_ptr<POD_pair<size_t,size_t> >::iterator iter = pair_stack.begin();
	const autovalarray_ptr<POD_pair<size_t,size_t> >::iterator iter_end = pair_stack.end();
	if (iter!=iter_end)
		do	{
			assert(iter->first<iter->second);
			if (target.first<iter->first)
				{
				if (target.second<iter->first) iter->first -= 1;
				iter->first -= 1;
				}
			if (target.first<iter->second)
				{
				if (target.second<iter->second) iter->second -= 1;
				iter->second -= 1;
				}
			}
		while(++iter!=iter_end);
}

static void balanced_character_kill(Token<char>& x, autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> >& pretokenized, autovalarray_ptr<POD_pair<size_t,size_t> >& parenpair_stack, size_t target_idx)
{
	assert(parenpair_stack.size()>target_idx);
	POD_pair<size_t,size_t> target	= parenpair_stack[target_idx];
	assert(pretokenized.size()>target.first);
	assert(pretokenized.size()>target.second);
	assert(target.first<target.second);
	assert(1==pretokenized[target.first].second);
	assert(1==pretokenized[target.second].second);
	{
	char* const tmp = x.c_array();
	tmp[pretokenized[target.first].first] = ' ';
	tmp[pretokenized[target.second].first] = ' ';
	}
	parenpair_stack.DeleteIdx(target_idx);
	pretokenized.DeleteIdx(target.second);
	pretokenized.DeleteIdx(target.first);
	pairstack_clean(target,parenpair_stack);
}

template<char c>
static inline bool
token_is_char(const char* const x, const POD_triple<size_t,size_t,lex_flags>& lexed_token)
{
	assert(x);
	return 1==lexed_token.second && c==x[lexed_token.first];
}

template<>
inline bool
token_is_char<'#'>(const char* const x, const POD_triple<size_t,size_t,lex_flags>& lexed_token)
{
	assert(x);
	return detect_C_stringize_op(x+lexed_token.first,lexed_token.second);
}

template<>
inline bool
token_is_char<'['>(const char* const x, const POD_triple<size_t,size_t,lex_flags>& lexed_token)
{
	assert(x);
	return detect_C_left_bracket_op(x+lexed_token.first,lexed_token.second);
}

template<>
inline bool
token_is_char<']'>(const char* const x, const POD_triple<size_t,size_t,lex_flags>& lexed_token)
{
	assert(x);
	return detect_C_right_bracket_op(x+lexed_token.first,lexed_token.second);
}

template<>
inline bool
token_is_char<'{'>(const char* const x, const POD_triple<size_t,size_t,lex_flags>& lexed_token)
{
	assert(x);
	return detect_C_left_brace_op(x+lexed_token.first,lexed_token.second);
}

template<>
inline bool
token_is_char<'}'>(const char* const x, const POD_triple<size_t,size_t,lex_flags>& lexed_token)
{
	assert(x);
	return detect_C_right_brace_op(x+lexed_token.first,lexed_token.second);
}

// Closely related to _C99_literal_converts_to_bool/CSupport.cpp
/*! 
 * examines a proposed token for whether it is suitable for an #if/#elif control expression, and if so 
 * 
 * \param x #if/#elif line
 * \param lexed_token where the token is
 * \param is_zero if suitable, set this to true iff token is 0 for purposes of preprocessing
 * 
 * \return true iff suitable for an #if/#elif control expression
 */
static bool if_elif_control_is_zero(const Token<char>& x, const POD_triple<size_t,size_t,lex_flags>& lexed_token, bool& is_zero)
{
	const lex_flags flags = lexed_token.third;
	if (C_TESTFLAG_CHAR_LITERAL==flags)
		{	//! \test if.C99/Pass_zero.hpp, if.C99/Pass_zero.h
			//! \test if.C99/Pass_nonzero.hpp, if.C99/Pass_nonzero.h
		is_zero = CCharLiteralIsFalse(x.data()+lexed_token.first,lexed_token.second);
		return true;
		};

	//! \todo --do-what-i-mean will handle floats as well
	if (!(C_TESTFLAG_PP_NUMERAL & flags)) return false;
	C_REALITY_CHECK_PP_NUMERAL_FLAGS(flags);
	if (C_TESTFLAG_FLOAT & flags) return false;
	// zeros go to zero, everything else canonicalizes to one
	//! \test if.C99/Pass_if_zero.hpp, if.C99/Pass_if_zero.h
	is_zero = C99_integer_literal_is_zero(x.data()+lexed_token.first,lexed_token.second,flags);
	return true;
}

/*! 
 * 
 * 
 * \param x overall directive
 * \param pretokenized token index for overall directive
 * \param src
 * \param i start token
 * \param delta last token offset included in range to be substituted
 * 
 * \return bool true if and only if directive obviously completely substituted
 */
bool
CPreprocessor::replace_char_into_directive(Token<char>& x,const autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> >& pretokenized,const char src,const size_t i,const size_t delta)
{
	if (0==i && delta+1==pretokenized.size())
		{
		x.replace_once(std::nothrow,pretokenized[0].first,x.size()-pretokenized[0].first,src);
		return true;
		};

	size_t buffer_used = 0;
	char Buffer[4] = "";
	if (0<pretokenized[i].first && lang.require_padding(x.data()[pretokenized[i].first-1],src))
		Buffer[buffer_used++] = ' ';
	Buffer[buffer_used++] = src;
	if (x.size()>pretokenized[i+delta].first+pretokenized[i+delta].second && lang.require_padding(src,x.data()[pretokenized[i+delta].first+pretokenized[i+delta].second]))
		Buffer[buffer_used++] = ' ';

	x.replace_once(std::nothrow,pretokenized[i].first,(pretokenized[i+delta].first-pretokenized[i].first)+pretokenized[i+delta].second,Buffer);
	return false;
}

/*
 * we use goto in CPreprocessor::if_elif_syntax_ok contrary to readable style guidelines
 *	RetryStringMerge: restart the string-merge stage (should happen only if running short on memory
 *	OneTokenExit: centralize exit code when down to one token
 */
bool
CPreprocessor::if_elif_syntax_ok(Token<char>& x, const autovalarray_ptr<char*>& macros_object, const autovalarray_ptr<Token<char>*>& macros_object_expansion, const autovalarray_ptr<char*>& macros_function, const autovalarray_ptr<Token<char>*>& macros_function_arglist, const autovalarray_ptr<Token<char>*>& macros_function_expansion,const type_system& min_types)
{
	const unsigned int if_directive = UNPACK_DIRECTIVE(x.flags);
	assert(PP::IF==if_directive || PP::ELIF==if_directive);
	//! \test if.C99/Error_no_arg.hpp, if.C99/Error_no_arg.h : #if no control expression
	//! \test if.C99/Error_if0_no_arg_elif.hpp, if.C99/Error_if0_no_arg_elif.h : #elif no control expression, critical
	//! \test if.C99/Error_if1_no_arg_elif.hpp, if.C99/Error_if1_no_arg_elif.h : #elif no control expression, non-critical; could allow this with a do-what-i-mean option
	if (!strcmp(x.data()+1,valid_directives[if_directive].first))
		{
		message_header(x);
		INC_INFORM(ERR_STR);
		INC_INFORM("#");
		INC_INFORM(valid_directives[if_directive].first);
		INFORM(" has no control expression. (C99 6.10p1/C++98 16p1)");
		zcc_errors.inc_error();
		return false;
		};

	// tokenize the whole line
	autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> > pretokenized;
	const size_t critical_offset = valid_directives[if_directive].second+2;
	//! \test if.C99/Error_control2.hpp, if.C99/Error_control2.h (#if)
	//! \todo __VA_ARGS__ within defined operator should only be a warning with --do-what-i-mean option
	bool bad_control = C99_VA_ARGS_flinch(x,critical_offset);

	//! \todo decide what to do when more languages are available for preprocessing
	lang.line_lex(x.data()+critical_offset,x.size()-critical_offset,pretokenized);
	STL_translate_first(critical_offset,pretokenized);	// coordinate fixup

	// analyze the defined operators
	size_t i = 0;
	do	if ((sizeof("defined")-1)==pretokenized[i].second && !strncmp(x.data()+pretokenized[i].first,"defined",(sizeof("defined")-1)))
			{
			if (i+1>=pretokenized.size())
				{	//! \test cpp/defined.C99/Error_malformed1.hpp, cpp/defined.C99/Error_malformed1.h
				message_header(x);
				INC_INFORM(ERR_STR);
				INC_INFORM("malformed defined operator application (C99 6.10.1p1/C++98 16.1p1)");
				INC_INFORM(x.data()+pretokenized[i].first,pretokenized[i].second);
				zcc_errors.inc_error();
				bad_control = true;
				break;
				};
			if (C_TESTFLAG_IDENTIFIER==pretokenized[i+1].third)
				{	// defined IDENTIFIER -- will evaluate to 0 or 1
					//! \test cpp/defined.C99/Pass_defined_op.hpp, cpp/defined.C99/Pass_defined_op.h
				int know_it_now = context_free_defined(x.data()+pretokenized[i+1].first, pretokenized[i+1].second);
				if (0==know_it_now)
					know_it_now = (macro_is_defined(x.data()+pretokenized[i+1].first, pretokenized[i+1].second, macros_object, macros_function)) ? 1 : -1;
				const char subst_dest = (0<know_it_now) ? '1' : '0';
				if (replace_char_into_directive(x,pretokenized,subst_dest,i,1)) return true;
				lang.line_lex(x.data()+critical_offset,x.size()-critical_offset,pretokenized);
				STL_translate_first(critical_offset,pretokenized);	// coordinate fixup
				continue;
				}
			else if (token_is_char<'('>(x.data(),pretokenized[i+1]))
				{	// defined(IDENTIFIER) [hopefully]
				if (i+3>=pretokenized.size())
					{	//! \test defined.C99/Error_malformed3.hpp, defined.C99/Error_malformed3.h
					message_header(x);
					INC_INFORM(ERR_STR);
					INC_INFORM("malformed defined operator application  (C99 6.10.1p1/C++98 16.1p1)");
					INC_INFORM(x.data()+pretokenized[i].first,pretokenized.back().second+(pretokenized.back().first-pretokenized[i].first));
					zcc_errors.inc_error();
					bad_control = true;
					break;
					}
				if (   C_TESTFLAG_IDENTIFIER==pretokenized[i+2].third
					&& token_is_char<')'>(x.data(),pretokenized[i+3]))
					{	// defined(IDENTIFIER)
						//! \test cpp/defined.C99/Pass_defined_op.hpp, cpp/defined.C99/Pass_defined_op.h
					int know_it_now = context_free_defined(x.data()+pretokenized[i+2].first, pretokenized[i+2].second);
					if (0==know_it_now)
						know_it_now = (macro_is_defined(x.data()+pretokenized[i+2].first, pretokenized[i+2].second, macros_object, macros_function)) ? 1 : -1;

					const char subst_dest = (0<know_it_now) ? '1' : '0';
					if (replace_char_into_directive(x,pretokenized,subst_dest,i,3)) return true;
					lang.line_lex(x.data()+critical_offset,x.size()-critical_offset,pretokenized);
					STL_translate_first(critical_offset,pretokenized);	// coordinate fixup
					continue;
					};
				//! \test defined.C99/Error_nonidentifier.hpp, defined.C99/Error_nonidentifier.h
				//! \test defined.C99/Error_excess_tokens.hpp, defined.C99/Error_excess_tokens.h
				message_header(x);
				INC_INFORM(ERR_STR);
				INC_INFORM("malformed defined operator application  (C99 6.10.1p1/C++98 16.1p1)");
				INC_INFORM(x.data()+pretokenized[i].first,pretokenized[i+3].second+(pretokenized[i+3].first-pretokenized[i].first));
				zcc_errors.inc_error();
				bad_control = true;
				continue;
				};
			//! \test defined.C99/Error_malformed2.hpp, defined.C99/Error_malformed2.h
			message_header(x);
			INC_INFORM(ERR_STR);
			INC_INFORM("malformed defined operator application  (C99 6.10.1p1/C++98 16.1p1)");
			INC_INFORM(x.data()+pretokenized[i].first,pretokenized[i+1].second+(pretokenized[i+1].first-pretokenized[i].first));
			zcc_errors.inc_error();
			bad_control = true;
			continue;
			}
	while(pretokenized.size() > ++i);

	// if any identifiers survive, we have to do macro preprocessing to get further
	if (bad_control) return false;	// but if we discard the block anyway it doesn't matter

	// analyze the __has_include() extension operator (from CLang)
	i = 0;
	do	if ((sizeof("__has_include")-1)==pretokenized[i].second && !strncmp(x.data()+pretokenized[i].first,"__has_include",(sizeof("__has_include")-1)))
			{
			// if no space for extension, let default flush-to-zero happen silently
			if (4>pretokenized.size()-i) break;
			if (!token_is_char<'('>(x.data(),pretokenized[i+1]))
				// do not trigger __has_include extension
				continue;
			// we want: __has_include("...") or __has_include(<...>)
			if (   C_TESTFLAG_STRING_LITERAL==pretokenized[i+2].third
				&& '"'==x.data()[pretokenized[i+2].first]
				&& token_is_char<')'>(x.data(),pretokenized[i+3]))
				{	// __has_include("...")
					//! \todo need test cases
				if (0==pretokenized[i+2].second-2)
					{	// empty, automatic failure
					if (replace_char_into_directive(x,pretokenized,'0',i,3)) return true;
					lang.line_lex(x.data()+critical_offset,x.size()-critical_offset,pretokenized);
					STL_translate_first(critical_offset,pretokenized);	// coordinate fixup
					continue;
					}

				char buf[FILENAME_MAX];
				char* const look_for = C_make_string(x.data()+pretokenized[i+2].first+1,pretokenized[i+2].second-2);
				if (strchr(look_for,'"'))
					{
					message_header(x);
					INFORM("__has_include(\"...\") contains \"; preprocessing per standards rather than invoking extension __has_include");
					free(look_for);
					i += 2;
					continue;
					};
				const char subst_dest = (detect_hardcoded_system_header(look_for,lang_code) || find_local_include(look_for, buf) || find_system_include(look_for, buf)) ? '1' : '0';
				free(look_for);
				if (replace_char_into_directive(x,pretokenized,subst_dest,i,3)) return true;
				lang.line_lex(x.data()+critical_offset,x.size()-critical_offset,pretokenized);
				STL_translate_first(critical_offset,pretokenized);	// coordinate fixup
				continue;
				}
			//! \todo: optimization check: does testing for < and > by leading/trailing character reduce code size
			else if (token_is_char<'<'>(x.data(),pretokenized[i+2]))
				{
				size_t j = i+2;
				while(pretokenized.size()-1 > ++j)
					if (token_is_char<'>'>(x.data(),pretokenized[j]))
						{	//! \todo need test cases
						if (!token_is_char<')'>(x.data(),pretokenized[j+1])) break;
						// __has_include(<...>)
						if (0==pretokenized[j].first-pretokenized[i+2].first+pretokenized[j].second-2)
							{	// empty, automatic failure
							if (replace_char_into_directive(x,pretokenized,'0',i,j+1-i)) return true;
							lang.line_lex(x.data()+critical_offset,x.size()-critical_offset,pretokenized);
							STL_translate_first(critical_offset,pretokenized);	// coordinate fixup
							continue;
							}
						char buf[FILENAME_MAX];
						char* const look_for = C_make_string(x.data()+pretokenized[i+2].first+1,pretokenized[j].first-pretokenized[i+2].first+pretokenized[j].second-2);
						if (strchr(look_for,'<'))
							{
							message_header(x);
							INFORM("__has_include(<...>) contains <; preprocessing per standards rather than invoking extension __has_include");
							free(look_for);
							i += 2;
							continue;
							};
						if (strchr(look_for,'>'))
							{
							message_header(x);
							INFORM("__has_include(<...>) contains >; preprocessing per standards rather than invoking extension __has_include");
							free(look_for);
							i += 2;
							continue;
							};
						const char subst_dest = (detect_hardcoded_system_header(look_for,lang_code) || find_system_include(look_for, buf)) ? '1' : '0';
						free(look_for);
						if (replace_char_into_directive(x,pretokenized,subst_dest,i,j+1-i)) return true;
						lang.line_lex(x.data()+critical_offset,x.size()-critical_offset,pretokenized);
						STL_translate_first(critical_offset,pretokenized);	// coordinate fixup
						continue;
						}
				++i;
				continue;
				}
			// not recognized as __has_include directive, preprocess per standards
			}
	while(pretokenized.size() > ++i);

	// if any identifiers survive, we have to do macro preprocessing to get further
	if (bad_control) return false;	// but if we discard the block anyway it doesn't matter

	intradirective_preprocess(x,valid_directives[if_directive].second+2,macros_object,macros_object_expansion,macros_function,macros_function_arglist,macros_function_expansion);

	// if we went invalid do not proceed further
	if (x.flags & INVALID_DIRECTIVE_FLAG) return false;

	// Hmm...intradirective_flush_identifiers_to_zero already called, this is redundant?
	intradirective_flush_identifiers_to_zero(x,valid_directives[if_directive].second+2);

	// ZCC should actually calculate the control expression completely to 0/1  Error reporting for this isn't particularly compatible with destructive-evaluate.
	// this is for both #if and #elif directives at depth 1
	// cf. USENET discussion with Eric Sosman @ sun.com on comp.lang.c at Rationale.XXX/comp.lang.c_must_fully_process_depth1_elif_20080827.htm
	// note that integer literals are always parsed as of type intmax_t or uintmax_t
	lang.line_lex(x.data()+critical_offset,x.size()-critical_offset,pretokenized);
	STL_translate_first(critical_offset,pretokenized);	// coordinate fixup

	{	// error the illegal preprocessing tokens here
	const size_t old_err_count = zcc_errors.err_count();
#ifdef NO_LEGACY_FIELDS
	const auto f_src = x.src->generic_string();
	const char* const f_name = f_src.c_str();
#else
	const char* const f_name = x.src_filename.data();
#endif
	i = pretokenized.size();
	do	{
		--i;
		lang.pp_support->AddPostLexFlags(x.data()+pretokenized[i].first, pretokenized[i].second, pretokenized[i].third, f_name, x.original_line.first);
		if (	(C_TESTFLAG_PP_OP_PUNC & pretokenized[i].third)
			&& 	((C_DISALLOW_POSTPROCESSED_SOURCE | C_DISALLOW_CONSTANT_EXPR | C_DISALLOW_IF_ELIF_CONTROL) & lang.pp_support->GetPPOpPuncFlags(C_PP_DECODE(pretokenized[i].third))))
			{
			message_header(x);
			INC_INFORM(ERR_STR);
			INC_INFORM("Forbidden token ");
			INC_INFORM(x.data()+pretokenized[i].first, pretokenized[i].second);
			INFORM(" in control expression.");
			zcc_errors.inc_error();
			};
		}
	while(0<i);
	if (old_err_count!=zcc_errors.err_count()) return false;
	}

	// should be merging string literals at this point
RetryStringMerge:
	bool want_clean_ws = false;
	bool str_concat_wants_RAM = false;
	if (2<=pretokenized.size())
		{	//! \bug need test cases...lots of them
		size_t i = 0;
		while(i<pretokenized.size())
			{
			if (C_TESTFLAG_STRING_LITERAL!=pretokenized[i+1].third)
				{
				i += 2;
				continue;
				}
			if (C_TESTFLAG_STRING_LITERAL!=pretokenized[i].third)
				{
				++i;
				continue;
				}

			char* test = NULL;
			const int res_code = lang.pp_support->EscapedStringConcatenate(x.data()+pretokenized[i].first,pretokenized[i].second,x.data()+pretokenized[i+1].first,pretokenized[i+1].second,test);
			assert(-5<=res_code && 1>=res_code);
			assert(1==res_code || NULL==test);
			assert(1!=res_code || NULL!=test);
			switch(res_code)
			{
			case -5:	{	// intermediate memory failed
						str_concat_wants_RAM = true;
			case 0:		++i;		// something exotic, mergence won't work
						continue;
						}
			case -4:	{	// second string literal content, but wide
						const size_t delta = pretokenized[i+1].first-(pretokenized[i].first+1U);
						x.intradelete(pretokenized[i].first+1U,delta);
						pretokenized[i].second = pretokenized[i+1].second+1;
						pretokenized.DeleteIdx(i+1);
						STL_translate_second(delta,pretokenized.begin()+i+1,pretokenized.end());
						continue;
						};
			case -3:	{	// first string literal content, but wide
							// convolution will handle UNICODE and raw strings as well
						const char width_tag = x.data()[pretokenized[i+1].first];
						memset(x.c_array()+pretokenized[i+1].first,' ',pretokenized[i+1].second);
						pretokenized.DeleteIdx(i+1);
						memmove(x.c_array()+pretokenized[i].first+1,x.c_array()+pretokenized[i].first,pretokenized[i].second);
						x.c_array()[pretokenized[i].first] = width_tag;
						++(pretokenized[i].first);
						want_clean_ws = true;
						continue;
						};
			case -2:	{	// second string literal is sufficient
						memset(x.c_array()+pretokenized[i].first,' ',pretokenized[i].second);
						pretokenized.DeleteIdx(i);
						want_clean_ws = true;
						continue;
						};
			case -1:	{	// first string literal is sufficient
						memset(x.c_array()+pretokenized[i+1].first,' ',pretokenized[i+1].second);
						pretokenized.DeleteIdx(i+1);
						want_clean_ws = true;
						continue;
						};
			case 1:		{	// no fancy memory conservation
						const size_t replace_span = (pretokenized[i+1].first-pretokenized[i].first)+pretokenized[i+1].second;
						const size_t core_str_target_len = strlen(test);
						assert(IsLegalCString(test,core_str_target_len));
						x.replace_once(std::nothrow,pretokenized[i].first,replace_span,test);
						pretokenized.DeleteIdx(i+1);
						if (core_str_target_len!=replace_span)
							STL_translate_second(core_str_target_len-replace_span,pretokenized.begin()+i+1,pretokenized.end());	// C99 requires wraparound for unsigned ints, so this works
						pretokenized[i].second = core_str_target_len;
						free(test);
						continue;
						};
			}
			}
		};
	if (want_clean_ws)
		{	// conserve memory
		if (_flush_token_gaps(x,pretokenized,lang) && str_concat_wants_RAM) goto RetryStringMerge;
		want_clean_ws = false;
		};

	// if there is only one token: it should be a preprocessing number or a character literal.  Handle it or error here
	if (1==pretokenized.size())
		{
oneTokenExit:
		assert(!str_concat_wants_RAM);
		bool is_zero = false;
		if (!if_elif_control_is_zero(x,pretokenized.front(),is_zero))
			{
			message_header(x);
			INC_INFORM(ERR_STR);
			INC_INFORM(x.data()+pretokenized.front().first,pretokenized.front().second);
			INFORM(" : value of control expression for #if/#elif must be a valid preprocessing integer (C99 6.10.1p1/C++98 16.1p1)");
			zcc_errors.inc_error();
			return false;
			}
		//! \todo --do-what-i-mean doesn't call this to evoke an error
#ifdef NO_LEGACY_FIELDS
		const auto f_src = x.src->generic_string();
		const char* const f_name = f_src.c_str();
#else
		const char* const f_name = x.src_filename.data();
#endif
		C99_literal_is_legal(x.data()+pretokenized.front().first,pretokenized.front().second,pretokenized.front().third, f_name,x.logical_line.first,min_types);
		x.replace_once(std::nothrow,critical_offset,x.size()-critical_offset,(is_zero) ? '0' : '1');
		return true;
		}

	{	// check for balancing errors (slow)
	autovalarray_ptr<weak_token> weaktoken_list;
	_weak_tokenize_aux(x,pretokenized,weaktoken_list);
	// do not process further if there are context-free errors
	if (lang.pp_support->BalancingErrorCheck(weaktoken_list.data(),weaktoken_list.size(),true,true)) return false;
	};

	autovalarray_ptr<POD_pair<size_t,size_t> > parenpair_stack;
	// context-free check should be intercepting when these are invalid
	construct_matched_pairs<'(',')'>(x,pretokenized,parenpair_stack);

	while(!parenpair_stack.empty() && 0==parenpair_stack.back().first && pretokenized.size()-1==parenpair_stack.back().second)
		{
		balanced_character_kill(x,pretokenized,parenpair_stack,parenpair_stack.size()-1);	// outermost parentheses pair: erase
		want_clean_ws = true;
		}

	// directly nested parentheses cleanup
	i = parenpair_stack.size();
	while(1<i)
		{
		--i;
		if (   parenpair_stack[i-1].first   ==parenpair_stack[i].first+1
			&& parenpair_stack[i-1].second+1==parenpair_stack[i].second)
			{
			balanced_character_kill(x,pretokenized,parenpair_stack,i);	// outer pair immediately contains inner pair
			want_clean_ws = true;
			}
		};
	// only-literal nested parentheses cleanup
	// valid only because no functions present or macros present [no identifiers left]
	i = parenpair_stack.size();
	while(0<i)
		{
		--i;
		if (   2==parenpair_stack[i].second-parenpair_stack[i].first
			&& ((C_TESTFLAG_CHAR_LITERAL | C_TESTFLAG_STRING_LITERAL | C_TESTFLAG_PP_NUMERAL) & pretokenized[parenpair_stack[i].first+1].third))
			{
			balanced_character_kill(x,pretokenized,parenpair_stack,i);
			want_clean_ws = true;
			}
		};

	if (pretokenized.empty())
		{	// if.C99/Error_control24.hpp, if.C99/Error_control24.h
		message_header(x);
		INC_INFORM(ERR_STR);
		INC_INFORM(x.data()+critical_offset,x.size()-critical_offset);
		INFORM(" : control expression for #if/#elif must evaluate to a single integer constant (C99 6.10.1p1/C++98 16.1p1)");
		zcc_errors.inc_error();
		return false;
		}

	// if there is only one token: it should be a preprocessing number or a character literal.  Handle it or error here
	if (1==pretokenized.size()) goto oneTokenExit;

	if (want_clean_ws)
		{
		if (_flush_token_gaps(x,pretokenized,lang) && str_concat_wants_RAM) goto RetryStringMerge;
		want_clean_ws = false;
		}

	{	// check for context-free errors
	autovalarray_ptr<weak_token> weaktoken_list;
	_weak_tokenize_aux(x,pretokenized,weaktoken_list);
	// do not process further if there are context-free errors
	if (lang.pp_support->ControlExpressionContextFreeErrorCheck(weaktoken_list.data(),weaktoken_list.size(),true,true)) return false;
	};

	// unfortunately, we need a parse tree even for this: the ternary operator ? : means we can't actually evaluate everything promiscuously
	{
	parse_tree_class parsetree;
	_parsetreeize_aux(x,pretokenized,parsetree);
	assert(parsetree.is_raw_list() || parsetree.is_atomic());
	if (parsetree.is_raw_list() && !lang.pp_support->CondenseParseTree(parsetree,min_types)) return false;
	if (!parsetree.is_atomic() && !lang.pp_support->EvalParseTree(parsetree,min_types)) return false;
	lang.pp_support->PPHackTree(parsetree,min_types);
	// final, when above is working properly
	if (!parsetree.is_atomic())
		{
		message_header(x);
		INC_INFORM(ERR_STR);
		INC_INFORM(x.data()+critical_offset,x.size()-critical_offset);
		INFORM(" : control expression for #if/#elif must evaluate to a single integer constant (C99 6.10.1p1/C++98 16.1p1)");
		zcc_errors.inc_error();
		return false;
		}
	x.replace_once(std::nothrow,critical_offset,x.size()-critical_offset,parsetree.index_tokens[0].token.first,parsetree.index_tokens[0].token.second);
	lang.line_lex(x.data()+critical_offset,x.size()-critical_offset,pretokenized);
	STL_translate_first(critical_offset,pretokenized);	// coordinate fixup
#ifdef NO_LEGACY_FIELDS
	const auto f_src = x.src->generic_string();
	const char* const f_name = f_src.c_str();
#else
	const char* const f_name = x.src_filename.data();
#endif
	i = pretokenized.size();
	do	{
		--i;
		lang.pp_support->AddPostLexFlags(x.data() + pretokenized[i].first, pretokenized[i].second, pretokenized[i].third, f_name, x.original_line.first);
		}
	while(0<i);
	str_concat_wants_RAM = false;
	}
	if (1==pretokenized.size()) goto oneTokenExit;
	return true;
}

// \todo? don't undef these when testing tokenize-flow preprocessing
#undef PREPROCESSING_DIRECTIVE_FLAG
#undef PACK_DIRECTIVE
#undef UNPACK_DIRECTIVE
#undef MAX_PP_DIRECTIVE
#undef PP_INVALID
#undef SYNTAX_CHECKED_FLAG

void
CPreprocessor::predefined_macro_replacement(Token<char>& x, size_t critical_offset)
{
	assert(x.size()>critical_offset);
	lex_flags token_flags;
	while(x.size()>critical_offset)
		{
		const size_t token_len = lang.UnfilteredNextToken(x.data()+critical_offset,token_flags);
		if (    C_TESTFLAG_IDENTIFIER!=token_flags
			|| !predefined_macro_replace_once(x,critical_offset,token_len))
			critical_offset += token_len;
		}
}

/*! 
 * Replaces a predefined macro if it is at the exact location named.
 * 
 * \param x token containing candidate for predefined macro
 * \param critical_offset where the candidate is
 * \param token_len length of candidate
 * 
 * \return true iff replaced a predefined macro.  Updates critical_offset when returning true
 */
bool
CPreprocessor::predefined_macro_replace_once(Token<char>& x, size_t& critical_offset, const size_t token_len)
{
	assert(x.size()>critical_offset);
	assert(x.size()>=critical_offset+token_len);
	const errr macro_index = linear_find(x.data()+critical_offset,token_len,macro_identifier_default,macro_identifier_default_count);
	if (-1!=macro_index)
		{
		const char* macro_value = NULL;
		char buf[10];
#ifdef NO_LEGACY_FIELDS
		std::string tmpfile;
#else
		char file_buf[MAX_PATH+2];
#endif
		if (macro_identifier_default[macro_index].second)
			// value known, substitute in
			macro_value = macro_identifier_default[macro_index].second;
		// special
		else if (!strcmp(macro_identifier_default[macro_index].first,"__FILE__"))
			{
#ifdef NO_LEGACY_FIELDS
			assert(!x.src.empty() && !x.src->empty());
			tmpfile = "\"";
			tmpfile += x.src->generic_string();
			tmpfile += "\"";
			macro_value = tmpfile.c_str();
#else
			assert(!x.src_filename.empty());
			file_buf[0] = '"';
			strcpy(file_buf+1,x.src_filename.data());
			file_buf[1+strlen(x.src_filename.data())] = '"';
			file_buf[2+strlen(x.src_filename.data())] = '\0';
			macro_value = file_buf;
#endif
			}
		else if (!strcmp(macro_identifier_default[macro_index].first,"__LINE__"))
			macro_value = z_umaxtoa(x.logical_line.first,buf,10);
		else if (!strcmp(macro_identifier_default[macro_index].first,"__TIME__"))
			macro_value = time_buffer;
		else if (!strcmp(macro_identifier_default[macro_index].first,"__TIMESTAMP__"))
			macro_value = time_date_buffer;
		else if (!strcmp(macro_identifier_default[macro_index].first,"__COUNTER__"))
			macro_value = z_umaxtoa(counter_macro++,buf,10);
		else if (!strcmp(macro_identifier_default[macro_index].first,"__INCLUDE_LEVEL__"))
			macro_value = z_umaxtoa(include_level,buf,10);
		else{
			assert(!strcmp(macro_identifier_default[macro_index].first,"__DATE__"));
			macro_value = date_buffer;
			}
		_macro_replace(x,critical_offset,token_len,macro_value);
		return true;
		}
	return false;
}

void
CPreprocessor::_macro_replace(Token<char>& x, size_t& critical_offset, const size_t token_len,const char* const macro_value) const
{
	assert(x.size()>critical_offset);
	assert(x.size()-critical_offset>=token_len);
	assert(0<token_len);

	if (is_empty_string(macro_value))
		{
		if (x.size()-critical_offset<=token_len)
			x.rtrim(token_len);
		else if (0==critical_offset)
			x.ltrim(token_len);
		else if (lang.require_padding(x.data()[critical_offset-1],x.data()[critical_offset+token_len]))
			x.replace_once(std::nothrow,critical_offset,token_len,' ');
		else
			x.intradelete(critical_offset,token_len);
		return;
		}

	const bool pad_left = 0<critical_offset && lang.require_padding(x.data()[critical_offset-1],macro_value[0]);
	const bool pad_right = x.size()>critical_offset+token_len && lang.require_padding(macro_value[strlen(macro_value)-1],x.data()[critical_offset+token_len]);

	if (pad_right) x.replace_once(critical_offset+token_len,0,' ');
	x.replace_once(critical_offset,token_len,macro_value);
	if (pad_left) x.replace_once(critical_offset,0,' ');

	critical_offset += strlen(macro_value);
	if (pad_right) ++critical_offset;
	if (pad_left) ++critical_offset;
}


void
CPreprocessor::instantiate_function_macro_arguments(autovalarray_ptr<Token<char>*>& arguments, const Token<char>& arglist, size_t arg_count) const
{
	lex_flags scratch_flags;
	size_t paren_depth = 1;
	size_t var_origin = 0;
	size_t offset = 1;
	size_t count_args = 0;
	assert(2<=arglist.size());
	assert('('==arglist.front());
	assert(')'==arglist.back());
	if (2==arglist.size())
		{
		assert(0==arg_count);
		arguments.reset();
		return;
		};
	assert(0!=arg_count);
	assert(arg_count<arglist.size());
	arguments.resize(arg_count);
	while(offset<arglist.size())
		{
		if 		('('==arglist.data()[offset])
			{
			++paren_depth;
			++offset;
			}
		else if	(')'==arglist.data()[offset])
			{
			--paren_depth;
			if (0==paren_depth)
				{	// update var
				if (var_origin+1<offset)
					{
					const size_t skip_ws = strspn(arglist.data()+var_origin+1,lang.WhiteSpace);
					if (offset-var_origin+1>skip_ws)
						{
						arguments[count_args] = new Token<char>(arglist,var_origin+1+skip_ws,offset-var_origin-1-skip_ws,0);
						_flush_duplicated_ws(*arguments[count_args],lang);
						}
					};
				return;
				}
			++offset;
			}
		else if (','==arglist.data()[offset] && 1==paren_depth && count_args<arg_count-1)
			{	// update var
			if (var_origin+1<offset)
				{
				const size_t skip_ws = strspn(arglist.data()+var_origin+1,lang.WhiteSpace);
				if (offset-var_origin+1>skip_ws)
					{
					arguments[count_args] = new Token<char>(arglist,var_origin+1+skip_ws,offset-var_origin-1-skip_ws,0);
					_flush_duplicated_ws(*arguments[count_args],lang);
					}
				};
			var_origin = offset;
			++offset;
			++count_args;
			}
		else
			offset += lang.UnfilteredNextToken(arglist.data()+offset,scratch_flags);
		}
}

bool
CPreprocessor::dynamic_macro_replace_once(Token<char>& x, size_t& critical_offset, size_t token_len, const autovalarray_ptr<char*>& macros_object, const autovalarray_ptr<Token<char>*>& macros_object_expansion, const autovalarray_ptr<char*>& macros_function, const autovalarray_ptr<Token<char>*>& macros_function_arglist, const autovalarray_ptr<Token<char>*>& macros_function_expansion, autovalarray_ptr<char*>* const used_macro_stack)
{
	assert(x.size()>critical_offset);
	assert(x.size()-critical_offset>=token_len);
	assert(macros_object.size()==macros_object_expansion.size());
	assert(macros_function.size()==macros_function_expansion.size());
	const errr object_macro_index = binary_find(x.data()+critical_offset,token_len,macros_object);
	const errr function_macro_index = binary_find(x.data()+critical_offset,token_len,macros_function);
	assert(0>object_macro_index || 0>function_macro_index);
	if (0<=object_macro_index)
		{
		if (!macros_object_expansion[object_macro_index])
			{
			_macro_replace(x,critical_offset,token_len,"");
			return true;
			};
		if (nonrecursive_macro_replacement_list(macros_object_expansion[object_macro_index]->data()))
			{
			_macro_replace(x,critical_offset,token_len,macros_object_expansion[object_macro_index]->data());
			return true;
			};
		size_t test_critical_offset = 0;
		Token<char> Test(x);
		Test.lslice(critical_offset+token_len);
		Test.ltrim(critical_offset);
		_macro_replace(Test,test_critical_offset,token_len,macros_object_expansion[object_macro_index]->data());
		predefined_macro_replacement(Test,0);
		if (!used_macro_stack)
			{
			autovalarray_ptr<char*> macro_stack(1);
			macro_stack[0] = C_make_string(x.data()+critical_offset,token_len);
			intradirective_preprocess(Test,0,macros_object,macros_object_expansion,macros_function,macros_function_arglist,macros_function_expansion,&macro_stack);
			}
		else{
			used_macro_stack->insertNSlotsAt(1,used_macro_stack->size());
			used_macro_stack->back() = C_make_string(x.data()+critical_offset,token_len);
			intradirective_preprocess(Test,0,macros_object,macros_object_expansion,macros_function,macros_function_arglist,macros_function_expansion,used_macro_stack);
			used_macro_stack->DeleteIdx(used_macro_stack->size()-1);
			}
		_macro_replace(x,critical_offset,token_len,Test.data());
		return true;
		};
	if (0<=function_macro_index && x.size()>critical_offset+token_len && '('==x.data()[critical_offset+token_len])
		{
		assert(macros_function_arglist[function_macro_index]);
		assert('('==macros_function_arglist[function_macro_index]->front());
		assert(')'==macros_function_arglist[function_macro_index]->back());
		const size_t formal_arg_span = macros_function_arglist[function_macro_index]->size();
		const size_t formal_arg_count = (2<formal_arg_span) ? std::count(macros_function_arglist[function_macro_index]->begin(),macros_function_arglist[function_macro_index]->end(),',')+1 : 0;
		const bool formal_varadic = 5<=formal_arg_span && !strncmp(macros_function_arglist[function_macro_index]->data()+(formal_arg_span-4),"...",sizeof("...")-1);
		size_t arg_count = 0;
		const size_t arg_span = function_macro_invocation_argspan(x.data()+critical_offset+token_len,x.size()-(critical_offset+token_len),arg_count);
		if (0==arg_span)
			{	//! \test Error_macro_arglist1.hpp
			message_header(x);
			INC_INFORM(ERR_STR);
			INC_INFORM("macro ");
			INC_INFORM(x.data()+critical_offset,token_len);
			INFORM(" did not close its argument list in time. (C99 6.10p1/C++98 16.1p1)");
			zcc_errors.inc_error();
			x.flags |= INVALID_DIRECTIVE_FLAG;
			return false;
			}
		if (formal_arg_count>arg_count || (formal_arg_count<arg_count && !formal_varadic))
			{	//! \test Error_macro_arglist2.hpp
				//! \test Error_macro_arglist3.hpp
			message_header(x);
			INC_INFORM(ERR_STR);
			INC_INFORM("macro ");
			INC_INFORM(x.data()+critical_offset,token_len);
			INC_INFORM(" had ");
			INC_INFORM(arg_count);
			INC_INFORM(" argument");
			INC_INFORM((1==arg_count) ? "" : "s");
			INC_INFORM(", needed ");
			if (formal_varadic) INC_INFORM("at least ");
			INC_INFORM(formal_arg_count);
			INFORM(". (C99 6.10p1/C++0x 16.1p1)");
			zcc_errors.inc_error();
			x.flags |= INVALID_DIRECTIVE_FLAG;
			return false;
			}
		if (!macros_function_expansion[function_macro_index])
			{
			_macro_replace(x,critical_offset,token_len+arg_span,"");
			return true;
			};
		if (nonrecursive_macro_replacement_list(macros_function_expansion[function_macro_index]->data()))
			{
			_macro_replace(x,critical_offset,token_len+arg_span,macros_function_expansion[function_macro_index]->data());
			return true;
			};

		autovalarray_ptr<Token<char>*> formal_arguments;
		autovalarray_ptr<Token<char>*> actual_arguments;
		{
		Token<char> actual_pre_args(x,critical_offset+token_len,arg_span,0);
		instantiate_function_macro_arguments(actual_arguments,actual_pre_args,formal_arg_count);
		}
		//! \todo consider calling this once on-demand (space vs. time trade)
		instantiate_function_macro_arguments(formal_arguments,*macros_function_arglist[function_macro_index],formal_arg_count);
		// arguments should already be normal-formed coming out, with as many as requested
		assert(formal_arguments.size()==formal_arg_count);
		assert(actual_arguments.size()==formal_arg_count);
#ifndef NDEBUG
		{	// reality-check the formal argument list
		lex_flags identifier_flags;
		size_t j = formal_arg_count;
		do	{
			--j;
			assert(formal_arguments[j]);
			assert(0<formal_arguments[j]->size());
			const size_t identifier_len = lang.UnfilteredNextToken(formal_arguments[j]->data(),identifier_flags);
			assert(C_TESTFLAG_IDENTIFIER==identifier_flags);
			assert(identifier_len==formal_arguments[j]->size());
			}
		while(0<j);
		}
#endif
		// ... is used as parameter __VA_ARGS__
		//! __VA_ARGS__ occurs only in the replacement list of varadic function-like macros, so it is the proper variable name for that ...
		if (formal_varadic) formal_arguments.back()->replace_once(0,formal_arguments.back()->size(),"__VA_ARGS__");
		//! \todo should discard unused formal arguments and their parameter lists; not worth a warning, as there are a number of legitimate uses for discarding formal parameters
		Token<char> Test(*macros_function_expansion[function_macro_index]);
		if (!used_macro_stack)
			{
			autovalarray_ptr<char*> macro_stack(1);
			macro_stack[0] = C_make_string(x.data()+critical_offset,token_len);
			dynamic_function_macro_prereplace_once(macros_object, macros_object_expansion, macros_function, macros_function_arglist, macros_function_expansion, &macro_stack, formal_arguments, actual_arguments, Test);
			}
		else{
			used_macro_stack->insertNSlotsAt(1,used_macro_stack->size());
			used_macro_stack->back() = C_make_string(x.data()+critical_offset,token_len); 
			dynamic_function_macro_prereplace_once(macros_object, macros_object_expansion, macros_function, macros_function_arglist, macros_function_expansion, used_macro_stack, formal_arguments, actual_arguments, Test);
			used_macro_stack->DeleteIdx(used_macro_stack->size()-1);
			}
		_macro_replace(x,critical_offset,token_len+arg_span,Test.data());
		return true;
		};
	return false;
}

/*! 
 * Does a single concatenation of the tokens indicated
 * 
 * \param x : where concatenation is being done
 * \param pretokenized : pointer into C array where tokenization info is
 * 
 * \return true if and only if concatenation happened
 */
static bool _concatenate_single(Token<char>& x,const POD_triple<size_t,size_t,lex_flags>* pretokenized, LangConf& lang)
{
	assert(pretokenized);
	autovalarray_ptr_throws<char> new_token(pretokenized[0].second+pretokenized[2].second);
	strncpy(new_token.c_array(),x.data()+pretokenized[0].first,pretokenized[0].second);
	strncpy(new_token.c_array()+pretokenized[0].second,x.data()+pretokenized[2].first,pretokenized[2].second);
	lex_flags scratch_flags;
	const size_t new_token_len = lang.UnfilteredNextToken(new_token.data(),scratch_flags);
	if (new_token_len!=new_token.size())
		{	//! \test define.C99/Error_concatenate3.hpp, define.C99/Error_concatenate3.h
			//! \test define.C99/Error_concatenate4.hpp, define.C99/Error_concatenate4.h
		message_header(x);
		INC_INFORM(ERR_STR);
		INC_INFORM("## concatenation result ");
		INC_INFORM(new_token.data());
		INFORM(" is not a single token.  Defining undefined behavior as eliminating ## and continuing (C99 6.10.3.3p3/C++98 16.3.3p3)");
		zcc_errors.inc_error();

		const size_t offset = pretokenized[0].first+pretokenized[0].second;
		if (new_token_len==pretokenized[0].second)
			x.intradelete(offset,pretokenized[2].first-offset);
		else
			x.replace_once(offset,pretokenized[2].first-offset,' ');
		return false;
		};
	// splice it
	//! \test define.C99/Pass_concatenate1.hpp, define.C99/Pass_concatenate1.h
	//! \test define.C99/Pass_concatenate2.hpp, define.C99/Pass_concatenate2.h
	x.replace_once(pretokenized[0].first,(pretokenized[2].first-pretokenized[0].first)+pretokenized[2].second,new_token.data());
	return true;
}

static void remove_ws_from_token(Token<char>& x, const autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> >& pretokenized)
{
	// truncate everything past last token
	x.lslice(pretokenized.back().first+pretokenized.back().second);
	size_t i = pretokenized.size();
	if (2<=i)
		{
		--i;
		do	{
			--i;
			if (   pretokenized[i].first<pretokenized[i+1].first
				&& pretokenized[i].second<pretokenized[i+1].first-pretokenized[i].first)
				{
				const size_t ws_origin = pretokenized[i].first+pretokenized[i].second;
				x.replace_once(std::nothrow,ws_origin,pretokenized[i+1].first-ws_origin,' ');
				}
			}
		while(0<i);
		}

	// truncate before first token
	x.ltrim(pretokenized.front().first);
}

void
CPreprocessor::dynamic_function_macro_prereplace_once(const autovalarray_ptr<char*>& macros_object, const autovalarray_ptr<Token<char>*>& macros_object_expansion, const autovalarray_ptr<char*>& macros_function, const autovalarray_ptr<Token<char>*>& macros_function_arglist, const autovalarray_ptr<Token<char>*>& macros_function_expansion,autovalarray_ptr<char*>* const used_macro_stack, const autovalarray_ptr<Token<char>*>& formal_arguments, const autovalarray_ptr<Token<char>*>& actual_arguments, Token<char>& x)
{
	assert(!actual_arguments.empty());
	// deal with # operators before macro-replacing arguments
	autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> > pretokenized;
	if (lang.line_lex_find(x.data(),x.size(),"#",sizeof("#")-1,pretokenized) || lang.line_lex_find(x.data(),x.size(),"%:",sizeof("%:")-1,pretokenized))
		{
		size_t i = pretokenized.size();
		do	{
			const POD_triple<size_t,size_t,lex_flags>* const origin = pretokenized.data() + --i; 
			if (detect_C_stringize_op(x.data()+origin[0].first,origin[0].second))
				{
				assert(pretokenized.size()>i+1);
				assert(origin[0].first+origin[0].second==origin[1].first);
				const errr j = (C_TESTFLAG_IDENTIFIER==origin[0].third) ? linear_find_STL_deref2(x.data()+origin[1].first,origin[1].second,formal_arguments) : -1;
				assert(0<=j);

				{
				autovalarray_ptr<char> stringized_actual;	//! \todo inefficient, should stringize any parameter only once and reuse
				stringize(stringized_actual,actual_arguments[j]);
				// safe because narrow string
				x.replace_once(origin[0].first,origin[1].second+origin[0].second,stringized_actual.data());
				// XXX leave behind garbage in pretokenized
				}
				}
			}
		while(0<i);
		};

	// deal with empty-var ## operators
	size_t j = actual_arguments.size();
	do if (!actual_arguments[--j])
			{	// have a genuinely empty argument
			if (lang.line_lex_find(x.data(),x.size(),"##",sizeof("##")-1,pretokenized) || lang.line_lex_find(x.data(),x.size(),"%:%:",sizeof("%:%:")-1,pretokenized))
				{
				assert(!detect_C_concatenation_op(x.data()+pretokenized.front().first,pretokenized.front().second));
				assert(!detect_C_concatenation_op(x.data()+pretokenized.back().first, pretokenized.back().second));
				size_t k = pretokenized.size();
				if (3<=k)
					{
					do	{
						const POD_triple<size_t,size_t,lex_flags>* const origin = pretokenized.data() + --k; 
						const char* const x_data = x.data();
						if (C_TESTFLAG_NONATOMIC_PP_OP_PUNC==origin[0].third && detect_C_concatenation_op(x_data+origin[0].first,origin[0].second))
							{
							if (C_TESTFLAG_IDENTIFIER==origin[1].third)
								{
								const errr j2 = linear_find_STL_deref2(x_data+origin[1].first,origin[1].second,formal_arguments);
								if (0<=j2 && !actual_arguments[j2])
									{	// we matched an empty parameter (concatenation identity)
									const size_t replace_start = origin[-1].first+origin[-1].second;
									size_t replace_len = (origin[1].first-replace_start)+origin[1].second;
									if (x.size()>replace_start+replace_len)
										replace_len += strspn(x_data+replace_start+replace_len,lang.WhiteSpace+1);
									x.replace_once(replace_start,replace_len,' ');
									continue;
									}
								};
							if (C_TESTFLAG_IDENTIFIER==origin[-1].third)
								{	// don't need should_continue bypass for last check
								const errr j2 = linear_find_STL_deref2(x_data+origin[-1].first,origin[-1].second,formal_arguments);
								if (0<=j2 && !actual_arguments[j2])
									{
									const size_t replace_start = (0<k) ? origin[-2].first+origin[-2].second : 0U;
									size_t replace_len = (origin[0].first-replace_start)+origin[0].second;
									if (x.size()>replace_start+replace_len)
										replace_len += strspn(x_data+replace_start+replace_len,lang.WhiteSpace+1);
	
									x.replace_once(origin[-1].first,(origin[0].first-origin[-1].first)+origin[0].second,' ');
//									continue;
									}
								};
							}
						}
					while(0<k);
					};
				}
			break;
			}
	while(0<j);

	// macro-replace all arguments
	{
	Token<char>* const * const actual_begin = actual_arguments.begin();
	Token<char>* const * iter = actual_arguments.end();
	do	if (*(--iter))
			intradirective_preprocess(**iter,0,macros_object,macros_object_expansion,macros_function,macros_function_arglist,macros_function_expansion,used_macro_stack);
	while(actual_begin!=iter);
	}

	lang.line_lex(x.data(),x.size(),pretokenized);
	assert(!pretokenized.empty());
	assert(!detect_C_concatenation_op(x.data()+pretokenized.front().first,pretokenized.front().second));
	assert(!detect_C_concatenation_op(x.data()+pretokenized.back().first,pretokenized.back().second));

	j = pretokenized.size();
	do	{
		--j;
		errr j2 = linear_find_STL_deref2(x.data()+pretokenized[j].first,pretokenized[j].second,formal_arguments);
		if (0<=j2)
			{
			assert(actual_arguments[j2]);
			_macro_replace(x,pretokenized[j].first,pretokenized[j].second,actual_arguments[j2]->data());
			lang.line_lex(x.data(),x.size(),pretokenized);
			}
		else if (detect_C_concatenation_op(x.data()+pretokenized[j].first,pretokenized[j].second))
			{	// hmm...
			j2 = linear_find_STL_deref2(x.data()+pretokenized[j-1].first,pretokenized[j-1].second,formal_arguments);
			if (0<=j2)
				{	// oops....must do replacement *before* the ## concatenation
				assert(actual_arguments[j2]);
				assert(!actual_arguments[j2]->empty());
				autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> > pretokenized_alt;
				lang.line_lex(actual_arguments[j2]->data(),actual_arguments[j2]->size(),pretokenized_alt);
				_macro_replace(x,pretokenized[j-1].first,pretokenized[j-1].second,actual_arguments[j2]->data());
				lang.line_lex(x.data(),x.size(),pretokenized);
				j += pretokenized_alt.size()-1;
				assert(detect_C_concatenation_op(x.data()+pretokenized[j].first,pretokenized[j].second));
				};
			if (_concatenate_single(x,pretokenized.data()+(j-1),lang)) --j;
			}
		}
	while(0<j);
}

static void _complete_string_character_literal_define(Token<char>& x, const Token<char>& src, size_t critical_offset, size_t first_token_len,const char delim, const char* const end_error)
{
	if (delim==x.back()) return;
	message_header(src);
	INC_INFORM(ERR_STR);
	INC_INFORM("#define ");
	INC_INFORM(src.data()+critical_offset,first_token_len);
	INC_INFORM(" ends in an unterminated");
	if ('L'==x.front()) INC_INFORM(" wide");
	INFORM(end_error);
	zcc_errors.inc_error();
	x.append(delim);
}

/*! 
 * Puts a macro expansion into a standardized form so that strcmp is sufficient to determine equivalence.
 * 
 * \param x macro expansion to be normalized in-place
 * 
 * \return false if the macro ends in an unterminated (wide?) string/character literal.  [We terminate it here, but the caller should react later on.]
 */
void
CPreprocessor::normalize_macro_expansion(Token<char>& x, const Token<char>& src, size_t critical_offset, size_t first_token_len)
{
	if (x.empty()) return;
	size_t offset = 0;
	{	//! \test Pass5.hpp
	const size_t skip_ws = strspn(x.data(),lang.WhiteSpace+1);
	if (0<skip_ws)
		{
		x.ltrim(skip_ws);
		if (x.empty()) return;
		}
	}
	while(offset<x.size())
		{
		lex_flags token_flags;
		offset += lang.UnfilteredNextToken(x.data()+offset,token_flags);
		if (x.size()<=offset)
			{
			if 		(C_TESTFLAG_STRING_LITERAL==token_flags)
				//! \test define.C99/Error_unterminated3.hpp, define.C99/Error_unterminated3.h
				//! \test define.C99/Error_unterminated4.hpp, define.C99/Error_unterminated4.h
				_complete_string_character_literal_define(x,src,critical_offset,first_token_len,'"'," string literal.  Terminating. (C99 6.4.5p1/C++98 2.13.4)");
			else if (C_TESTFLAG_CHAR_LITERAL  ==token_flags)
				//! \test define.C99/Error_unterminated1.hpp, define.C99/Error_unterminated1.h
				//! \test define.C99/Error_unterminated2.hpp, define.C99/Error_unterminated2.h
				_complete_string_character_literal_define(x,src,critical_offset,first_token_len,'\''," character literal.  Terminating. (C99 6.4.4.4p1/C++98 2.13.2)");
			return;
			}
		const size_t skip_ws = strspn(x.data()+offset,lang.WhiteSpace+1);
		if (0<skip_ws)
			{
			if (x.size()-offset<=skip_ws)
				{	//! \test define.C99/Pass_dup5.hpp, define.C99/Pass_dup5.h
				x.lslice(offset);
				return;
				};
			//! \test define.C99/Pass_dup6.hpp, define.C99/Pass_dup6.h
			x.replace_once(std::nothrow,offset,skip_ws,' ');
			++offset;
			};
		};
	return;
}

void
CPreprocessor::intradirective_preprocess(Token<char>& x, size_t critical_offset, const autovalarray_ptr<char*>& macros_object, const autovalarray_ptr<Token<char>*>& macros_object_expansion, const autovalarray_ptr<char*>& macros_function, const autovalarray_ptr<Token<char>*>& macros_function_arglist, const autovalarray_ptr<Token<char>*>& macros_function_expansion,autovalarray_ptr<char*>* const used_macro_stack)
{
	while(x.size()>critical_offset)
		{
		const size_t skip_ws = strspn(x.data()+critical_offset,lang.WhiteSpace+1);
		if (0<skip_ws)
			{
			x.replace_once(std::nothrow,critical_offset,skip_ws,' ');
			++critical_offset;
			continue;
			};
		lex_flags token_flags;
		const size_t token_len = lang.UnfilteredNextToken(x.data()+critical_offset,token_flags);
		if (    C_TESTFLAG_IDENTIFIER!=token_flags
			|| (NULL!=used_macro_stack && 0<=binary_find(x.data()+critical_offset,token_len,*used_macro_stack))
			|| (	!predefined_macro_replace_once(x,critical_offset,token_len)
				&&	!dynamic_macro_replace_once(x,critical_offset,token_len,macros_object,macros_object_expansion,macros_function,macros_function_arglist,macros_function_expansion,used_macro_stack)))
			{
			critical_offset += token_len;
			continue;
			}
		};
}

void
CPreprocessor::intradirective_flush_identifiers_to_zero(Token<char>& x, size_t critical_offset) const
{	//! test if.C99/Error_if_control.hpp, if.C99/Error_if_control.h
	//! test if.C99/Pass_if_control.hpp, if.C99/Pass_if_control.h
	while(x.size()>critical_offset)
		{
		const size_t skip_ws = strspn(x.data()+critical_offset,lang.WhiteSpace+1);
		if (0<skip_ws)
			{
			x.replace_once(std::nothrow,critical_offset,skip_ws,' ');
			++critical_offset;
			continue;
			};
		lex_flags token_flags;
		const size_t token_len = lang.UnfilteredNextToken(x.data()+critical_offset,token_flags);
		if (C_TESTFLAG_IDENTIFIER!=token_flags)
			{
			critical_offset += token_len;
			continue;
			}
		_macro_replace(x,critical_offset,token_len,"0");
		critical_offset += 1;
		};
}

void CPreprocessor::die_on_pp_errors() const
{
	if (0>=zcc_errors.err_count()) return;
	INC_INFORM("FATAL: ");
	INC_INFORM(zcc_errors.err_count());
	INC_INFORM(" preprocessing error");
	INFORM((1==zcc_errors.err_count()) ? "\n" : "s\n");
	exit(EXIT_FAILURE);
}

void
CPreprocessor::debug_to_stderr(const autovalarray_ptr<Token<char>* >& TokenList,const autovalarray_ptr<char*>& macros_object, const autovalarray_ptr<Token<char>*>& macros_object_expansion, const autovalarray_ptr<char*>& macros_function, const autovalarray_ptr<Token<char>*>& macros_function_arglist, const autovalarray_ptr<Token<char>*>& macros_function_expansion,const autovalarray_ptr<char*>& locked_macros) const
{	// need whitespace tokens here to force pretty-printing
	if (!debug_mode) return;
	const size_t list_size = TokenList.size();
	size_t i = 0;
	while(i<list_size)
		{
#ifdef NO_LEGACY_FIELDS
		if (0<i && TokenList[i-1]->logical_line.first==TokenList[i]->logical_line.first && TokenList[i-1]->src == TokenList[i]->src && lang.require_padding(TokenList[i-1]->back(),TokenList[i]->front()))
#else
		if (0 < i && TokenList[i - 1]->logical_line.first == TokenList[i]->logical_line.first && !strcmp(TokenList[i - 1]->src_filename.data(), TokenList[i]->src_filename.data()) && lang.require_padding(TokenList[i - 1]->back(), TokenList[i]->front()))
#endif
			INC_INFORM(' ');

#ifdef NO_LEGACY_FIELDS
		if (list_size <= i + 1 || TokenList[i]->logical_line.first != TokenList[i + 1]->logical_line.first || TokenList[i]->src != TokenList[i + 1]->src)
#else
		if (list_size<=i+1 || TokenList[i]->logical_line.first!=TokenList[i+1]->logical_line.first || strcmp(TokenList[i]->src_filename.data(),TokenList[i+1]->src_filename.data()))
#endif
			INFORM(TokenList[i]->data());
		else
			INC_INFORM(TokenList[i]->data());
		++i;
		};
	// dump macros and locked macros
	INC_INFORM("#define __DATE__ ");
	INFORM(date_buffer);
	INC_INFORM("#define __TIME__ ");
	INFORM(time_buffer);
	INC_INFORM("#define __TIMESTAMP__ ");
	INFORM(time_date_buffer);
	i = 7;
	while(i<macro_identifier_default_count)
		{
		INC_INFORM("#define ");
		INC_INFORM(macro_identifier_default[i].first);
		INC_INFORM(" ");
		INFORM(macro_identifier_default[i].second);
		++i;
		};
	const size_t object_macro_size = macros_object.size();
	i = 0;
	while(i<object_macro_size)
		{
		INC_INFORM("#define ");
		INC_INFORM(macros_object[i]);
		if (macros_object_expansion[i])
			{
			INC_INFORM(" ");
			INFORM(macros_object_expansion[i]->data());
			}
		else
			INC_INFORM("\n");
		++i;
		}
	const size_t function_macro_size = macros_function.size();
	i = 0;
	while(i<function_macro_size)
		{
		INC_INFORM("#define ");
		INC_INFORM(macros_function[i]);
		INC_INFORM(macros_function_arglist[i]->data());
		if (macros_function_expansion[i])
			{
			INC_INFORM(" ");
			INFORM(macros_function_expansion[i]->data());
			}
		else
			INC_INFORM("\n");
		++i;
		}
	const size_t locked_macro_size = locked_macros.size();
	i = 0;
	while(i<locked_macro_size)
		{
		INC_INFORM("#pragma ZCC lock ");
		INFORM(locked_macros[i++]);
		};
}

/*! 
 * Causes a preprocessing error if the identifier __VA_ARGS__ is found.
 * 
 * \param x : token to examine for __VA_ARGS__
 * \param critical_offset : start point of examination
 * 
 * \return bool if and only if an error was found.
 */
bool
CPreprocessor::C99_VA_ARGS_flinch(const Token<char>& x, const size_t critical_offset) const
{	//! \todo option to bypass this
	assert(x.size()>critical_offset);
	if (SIZE_MAX==lang.lex_find(x.data()+critical_offset,x.size()-critical_offset,"__VA_ARGS__",sizeof("__VA_ARGS__")-1)) return false;
	message_header(x);
	INC_INFORM(ERR_STR);
	INFORM("identifier __VA_ARGS__ not allowed here.  Continuing. (C99 6.10.3p5/C++0x 16.3p5)");
	zcc_errors.inc_error();
	return true;
}

void
CPreprocessor::discard_duplicate_define(autovalarray_ptr<Token<char>* >& TokenList, const size_t i, const size_t critical_offset, const size_t first_token_len)
{
	message_header(*TokenList[i]);
	INC_INFORM(ERR_STR);
	INC_INFORM(TokenList[i]->data()+critical_offset,first_token_len);
	INFORM(" is already #define'd; discarding #define (C99 6.10.3p2/C++98 16.3p2,3)");
	TokenList.DeleteIdx(i);
	zcc_errors.inc_error();
}

/*! 
 * Discards leading and trailing ## operators in macro expansion lists; errors but continues.
 * 
 * \param x : macro expansion
 * 
 * \return bool : true iff truncated to empty
 */
bool CPreprocessor::discard_leading_trailing_concatenate_op(Token<char>& x)
{
	if ((sizeof("##")-1)>x.size()) return false;
	assert(!strpbrk(x.data(),lang.WhiteSpace+2));	// check for normalization
	assert(!strchr(x.data(),'\n'));	// check for normalization
	// tokenize the whole line
	autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> > pretokenized;
	lang.line_lex(x.data(),x.size(),pretokenized);

	if (detect_C_concatenation_op(x.data()+pretokenized.back().first,pretokenized.back().second))
		{
		message_header(x);
		INC_INFORM(ERR_STR);
		INFORM("concatenation operator ## ending a macro replacement list: removing and continuing (C99 6.10.3.3p1/C++98 16.3.3p1)");
		zcc_errors.inc_error();
		if (1==pretokenized.size()) return x.reset(),true;
		pretokenized.DeleteIdx(pretokenized.size()-1);
		x.lslice(pretokenized.back().first+pretokenized.back().second);
		}

	if (detect_C_concatenation_op(x.data()+pretokenized.front().first,pretokenized.front().second))
		{
		message_header(x);
		INC_INFORM(ERR_STR);
		INFORM("concatenation operator ## starting a macro replacement list: removing and continuing (C99 6.10.3.3p1/C++98 16.3.3p1)");
		zcc_errors.inc_error();
		if (1==pretokenized.size()) return x.reset(),true;
		x.ltrim(pretokenized[1].first);
		// XXX leave data structures in an inconsistent state: they'll be immediately destructed anyway
		}
	return false;
}

void
CPreprocessor::use_line_directive_and_discard(autovalarray_ptr<Token<char>* >& TokenList, const size_t i)
{
	assert(i<TokenList.size());
	assert(TokenList[i]);
	assert(!strncmp(TokenList[i]->data(),"#line ",sizeof("#line ")-1));
	if (TokenList[i]->flags & INVALID_DIRECTIVE_FLAG)
		{
		TokenList.DeleteIdx(i);
		return;
		};
	C_PPDecimalInteger line_number;
	lex_flags first_token_flags;
	lex_flags second_token_flags;
	size_t critical_offset = sizeof("#line ")-1;
	{	// C99: check for decimal integer literal, then optional string literal; error if this is not found
	const size_t first_token_len = lang.UnfilteredNextToken(TokenList[i]->data()+critical_offset,first_token_flags);
	if (     C_TESTFLAG_PP_NUMERAL!=first_token_flags
		||	!C_PPDecimalInteger::is(TokenList[i]->data()+critical_offset,first_token_len,line_number)
		||	-1==cmp(line_number,"1",sizeof("1")-1)
		||	 1==cmp(line_number,"2147483647",sizeof("2147483647")-1))	// constant should be stringized 2^31-1
		{	//! \test cpp/line.C99/Error_badnum1.h, cpp/line.C99/Error_badnum1.hpp
			//! \test cpp/line.C99/Error_badnum2.h, cpp/line.C99/Error_badnum2.hpp
			//! \test cpp/line.C99/Error_badnum3.h, cpp/line.C99/Error_badnum3.hpp
			//! \test cpp/line.C99/Error_badnum4.h, cpp/line.C99/Error_badnum4.hpp
		message_header(*TokenList[i]);
		INC_INFORM(ERR_STR);
		INFORM("#line does not have a line number between 1 and 2147483647 inclusive (C99 6.10.4p3/C++0x 16.4p3)");
		zcc_errors.inc_error();
		TokenList.DeleteIdx(i);
		return;
		};
	critical_offset += first_token_len;
	}
	if (i+1 < TokenList.size())
		{	//! \test cpp/line.C99/Preprocess_42.h, cpp/line.C99/Preprocess_42.hpp
			// Behavior is undefined if the syntax is bad (line number with malformed filename string), so don't worry about this happening too early.
		size_t numeric_line_number;
		{	// don't worry about failure (pre-validated above), *but* as errno should be a synchronized global we have to lock it anyway
		OS::scoped_lock tmp(errno_mutex);
		numeric_line_number = line_number.to_umax();	//! \warning can catastrophically overflow if size_t isn't at least 32-bit
		};
		if (TokenList[i]->logical_line.first!=numeric_line_number)
			{
			size_t j = i+1;
			//! \todo loops should stop at first (valid) #line directive
			// remember that unsigned arithmetic is modulo
			const size_t delta = numeric_line_number - TokenList[i]->logical_line.first;
			while(j<TokenList.size())
				TokenList[j++]->logical_line.first += delta;
			};
		}

	if (TokenList[i]->size()<=critical_offset)
		{	// just the line number needed updating
		TokenList.DeleteIdx(i);
		return;
		}

	{
	const size_t skip_ws = strspn(TokenList[i]->data()+critical_offset,lang.WhiteSpace+1);
	if (skip_ws>=TokenList[i]->size()-critical_offset)
		{	// only trailing whitespace
		TokenList.DeleteIdx(i);
		return;
		}
	critical_offset += skip_ws;
	}

	const size_t second_token_len = lang.UnfilteredNextToken(TokenList[i]->data()+critical_offset,second_token_flags);
	//! \todo: extension?  C99 spec requires narrow string literals only
	if (   C_TESTFLAG_STRING_LITERAL!=second_token_flags
		|| 'L'==TokenList[i]->data()[critical_offset])
		{	//! \test cpp/line.C99/Error_badfile1.h, cpp/line.C99/Error_badfile1.hpp
			//! \test cpp/line.C99/Error_badfile2.h, cpp/line.C99/Error_badfile2.hpp
		message_header(*TokenList[i]);
		INC_INFORM(ERR_STR);
		INFORM("#line wants a narrow string literal for the new __FILE__ (C99 6.10.4p1/C++98 16.4p1)");
		zcc_errors.inc_error();
		TokenList.DeleteIdx(i);
		return;
		}

	if (i+1 < TokenList.size())
		{	//! \test cpp/line.C99/Preprocess_42.h, cpp/line.C99/Preprocess_42.hpp
		//! \todo loops should stop at first (valid) #line directive with a filename
		// unescape the string, if needed
#ifdef NO_LEGACY_FIELDS
		decltype(TokenList[0]->src) new_FILE;
		const size_t escape_length = lang.UnescapeStringLength(TokenList[i]->data() + critical_offset + 1, second_token_len - 2);
		if (escape_length >= second_token_len - 2)
			new_FILE = canonical_cache<std::filesystem::path>::get().track(std::string(TokenList[i]->data() + critical_offset + 1, second_token_len - 2));
		else if (0 < escape_length) {
			autovalarray_ptr_throws<char> tmp(escape_length);
			lang.UnescapeString(tmp.c_array(), TokenList[i]->data() + critical_offset + 1, second_token_len - 2);
			new_FILE = canonical_cache<std::filesystem::path>::get().track(std::string(tmp.data(), tmp.size()));
		}

		size_t j = i + 1;
		if (new_FILE != TokenList[j]->src)
			while (j < TokenList.size())
				TokenList[j++]->src = new_FILE;
#else
		zaimoni::flyweight<const char> new_FILE;
		const size_t escape_length = lang.UnescapeStringLength(TokenList[i]->data()+critical_offset+1,second_token_len-2);
		if (escape_length>=second_token_len-2)
			new_FILE = C_string_to_flyweight(TokenList[i]->data()+critical_offset+1,second_token_len-2);
		else if (0<escape_length)
			{
			autovalarray_ptr_throws<char> tmp(escape_length);
			lang.UnescapeString(tmp.c_array(),TokenList[i]->data()+critical_offset+1,second_token_len-2);
			new_FILE = zaimoni::flyweight<const char>(tmp.release());
			}

		size_t j = i+1;
		if (strcmp(new_FILE.data(),TokenList[j]->src_filename.data()))
			while(j<TokenList.size())
				TokenList[j++]->src_filename = new_FILE;
#endif
		}

	TokenList.DeleteIdx(i);
}

// \todo? don't undef these when testing tokenize-flow preprocessing
#undef ULONG_BIT
#undef INVALID_DIRECTIVE_FLAG

void
CPreprocessor::truncate_illegal_tokens(Token<char>& x,const int directive_type,const size_t critical_offset)
{
	if (x.size()<=critical_offset) return;
	if (x.size()-critical_offset>strspn(x.data()+critical_offset,lang.WhiteSpace+1))
		{
		message_header(x);
		INC_INFORM(ERR_STR);
		INC_INFORM("#");
		INC_INFORM(valid_directives[directive_type].first);
		INFORM(" is trailed by illegal preprocessing tokens; discarding them. (C99 6.10p1/C++98 16p1)");
		zcc_errors.inc_error();
		}
	x.lslice(critical_offset);
}

bool
CPreprocessor::hard_locked_macro(const char* const x,const size_t x_len) const
{
	assert(x && *x);
	assert(0<x_len);
// C99: 6.11.9 Predefined macro names
// Macro names beginning with __STDC_ are reserved for future standardization.
//! \bug should have positive test suite for named __STDC_ macros
	if (7<=x_len && !strncmp(x,"__STDC_",sizeof("__STDC_")-1)) return true;
#ifndef BUILD_Z_CPP
// Lock down our relay identifiers. to be safe
	if (0<=linear_find_lencached(x,x_len,pragma_relay_keywords,STATIC_SIZE(pragma_relay_keywords))) return true;
#endif
// C++0x 17.4.3.2.2 simply prohibits all keywords as macros; prefer this to C++98.  C99/C0X is handled elsewhere, as it isn't so draconian.
// follow C++0x when generalizing to non-standard languages, as that's more intuitive.
	if (Lang::C!=lang_code && 0<=linear_find_lencached(x,x_len,lang.InvariantKeywords,lang.len_InvariantKeywords)) return true;
	return 0<=linear_find_lencached(x,x_len,macro_locked_default,macro_locked_default_count);
}

size_t CPreprocessor::function_macro_argument_span(const char* const x) const
{
	assert(!is_empty_string(x));
	if ('('!=x[0]) return 0;
	const size_t x_len = strlen(x);
	size_t span = 1;
	bool identifier_next = true;
	bool any_identifier = false;
	bool any_ellipsis = false;
	while(x_len>span)
		{
		if (lang.IsWS_NotFirst(x[span]))
			{	//! \test define.C99/Error_arglist4.hpp, define.C99/Error_arglist4.h
				//! \test define.C99/Error_arglist8.hpp, define.C99/Error_arglist8.h
			span += strspn(x+span,lang.WhiteSpace+1);
			continue;
			};
		if (')'==x[span])
			{	//! \test define.C99/Error_arglist5.hpp, define.C99/Error_arglist5.h
			if (!any_identifier || any_ellipsis || !identifier_next) return span+1;
			INFORM(") after , in function-like macro argument list.");
			return 0;
			}
		if (','==x[span])
			{
			if (identifier_next)
				{	//! \test define.C99/Error_arglist6.hpp, define.C99/Error_arglist6.h
				INFORM(", when identifier or ... expected in function-like macro argument list.");
				return 0;
				}
			if (any_ellipsis)
				{	//! \test define.C99/Error_arglist7.hpp, define.C99/Error_arglist7.h
				INFORM(", after ... in function-like macro argument list.");
				return 0;
				}
			identifier_next = true;
			++span;
			continue;
			}


		lex_flags scratch_flags;
		const size_t token_len = lang.UnfilteredNextToken(x+span,scratch_flags);
		if (C_TESTFLAG_IDENTIFIER==scratch_flags)
			{
			assert(0<token_len);
			if (identifier_next)
				{
				span += token_len;
				identifier_next = false;
				any_identifier = true;
				continue;
				};
			//! \test define.C99/Error_arglist8.hpp, define.C99/Error_arglist8.h
			INC_INFORM("Missing comma in function-like macro argument list before placeholder ");
			INC_INFORM(x+span,token_len);
			INFORM(".");
			return 0;
			}
		else if (C_TESTFLAG_NONATOMIC_PP_OP_PUNC==scratch_flags && 3==token_len && !strncmp(x+span,"...",3))
			{
			if (identifier_next)
				{	//! \test define.C99/Pass_arglist.hpp, define.C99/Pass_arglist.h
				span += token_len;
				identifier_next = false;
				any_ellipsis = true;
				continue;
				};
			//! \test define.C99/Error_arglist3.hpp, define.C99/Error_arglist3.h
			//! \test define.C99/Error_arglist4.hpp, define.C99/Error_arglist4.h
			INFORM("Missing comma in function-like macro argument list before ...");
			return 0;
			};
		//! \test define.C99/Error_arglist2.hpp, define.C99/Error_arglist2.h
		INC_INFORM("Unexpected token ");
		INC_INFORM(x+span,token_len);
		INFORM(" in function-like macro argument list.");
		return 0;
		};
	//! \test define.C99/Error_arglist1.hpp, define.C99/Error_arglist1.h
	return 0;
}

size_t
CPreprocessor::defined_span(const Token<char>& x, const size_t logical_offset, POD_pair<size_t,size_t>& identifier)
{	//! \todo more verbose error reporting
	assert(!strncmp(x.data()+logical_offset,"defined",(sizeof("defined")-1)));
	size_t offset = logical_offset;
	offset += (sizeof("defined")-1);
	if (x.size()<=offset) return 0;
	const size_t skip_ws = strspn(x.data()+offset,lang.WhiteSpace+1);
	offset += skip_ws;
	if (x.size()<=offset) return 0;
	const bool paren_delimited = ('('==x.data()[offset]);
	if (paren_delimited)
		{
		if (x.size()<= ++offset) return 0;
		}
	else if (0==skip_ws)
		return 0;

	lex_flags scratch_flags;
	const size_t token_len = lang.UnfilteredNextToken(x.data()+offset,scratch_flags);
	if (C_TESTFLAG_IDENTIFIER!=scratch_flags) return 0;
	identifier.first = offset;
	identifier.second = token_len;
	offset += token_len;
	if (x.size()<=offset) return 0;
	if (!paren_delimited) return offset-logical_offset;
	offset += strspn(x.data()+offset,lang.WhiteSpace+1);
	if (x.size()<=offset) return 0;
	if (')'==x.data()[offset]) return (offset-logical_offset)+1;
	return 0;
}

/*! 
 * Checks that a string has no identifiers within it.  That means no macro replacement can happen within it.
 *
 * \pre lang uses C_TESTFLAG_IDENTIFIER to indicate an identifier was lexed
 * 
 * \param x : string to be checked for identifiers
 * 
 * \return bool : true iff there are no identifiers within the string.
 */
bool
CPreprocessor::nonrecursive_macro_replacement_list(const char* const x) const
{
	if (is_empty_string(x)) return true;
	const size_t x_len = strlen(x);
	size_t offset = 0;
	lex_flags scratch_flags;
	while(x_len>offset)
		{
		const size_t token_len = lang.UnfilteredNextToken(x+offset,scratch_flags);
		if (C_TESTFLAG_IDENTIFIER==scratch_flags) return false;
		offset += token_len;
		};
	return true;
}

/*! 
 * Calculates whether a function macro invocation completely fits on the given string.
 * 
 * \param src		string to examine for argument list
 * \param src_span	length of string
 * \param arg_count	if return value non-zero, actual number of arguments found.
 * 
 * \return size_t	length of macro invocation argument list
 */
size_t
CPreprocessor::function_macro_invocation_argspan(const char* const src,const size_t src_span,size_t& arg_count) const
{
	assert(src);
	assert('('==src[0]);
	assert(src_span==strlen(src));
	if (2>src_span) return 0;
	if (')'==src[1]) return arg_count = 0,2; 
	lex_flags scratch_flags;
	size_t test_arg_count = 1;
	size_t paren_depth = 1;
	size_t i = 1;
	do	{
		if ('('==src[i]) ++paren_depth;
		else if (')'==src[i])
			{
			if (0== --paren_depth)
				{
				arg_count = test_arg_count;
				return i+1;
				};
			}
		else if (','==src[i])
			{
			if (1==paren_depth) ++test_arg_count;
			}
		else{
			const size_t token_len = lang.UnfilteredNextToken(src+i,scratch_flags);
			assert(0<token_len);
			i += (token_len-1);
			}
		}
	while(src_span > ++i);
	return 0;
}

/*! 
 * implements C preprocessor stringize operator.
 * 
 * \param dest valid C string representing src (postcondition)
 * \param src to be stringized
 */
void
CPreprocessor::stringize(autovalarray_ptr<char>& dest,const Token<char>* const & src)
{
	if (!src || src->empty())
		{	// empty string
		dest.resize(2);
		strcpy(dest.c_array(),"\"\"");
		return;
		}
	
	dest.resize(2+lang.EscapeStringLength(src->data(),src->size()));
	dest.front()='"';
	dest.back()='"';

	lang.EscapeString(dest.c_array()+1,src->data(),src->size());
}

/*! 
 * errors and removes clearly bad stringize operators
 * 
 * \param x : macro replacement list to verify
 * 
 * \return bool true if macro was emptied (caller should complete the cleanup)
 */
bool
CPreprocessor::flush_bad_stringize(Token<char>& x, const Token<char>& arglist)
{
	autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> > pretokenized;
	if (lang.line_lex_find(x.data(),x.size(),"#",sizeof("#")-1,pretokenized) || lang.line_lex_find(x.data(),x.size(),"%:",sizeof("%:")-1,pretokenized))
		{
		size_t i = 0;
		while(pretokenized.size()-1>i)
			{
			if (detect_C_stringize_op(x.data()+pretokenized[i].first,pretokenized[i].second))
				{
				if (pretokenized[i].first+pretokenized[i].second!=pretokenized[i+1].first)
					{	//! \test define.C99/Error_stringize2.hpp, define.C99/Error_stringize2.h
					message_header(x);
					INC_INFORM(ERR_STR);
					INFORM("# followed by whitespace; excising and continuing (C99 6.10.3.2p1/C++98 16.3.2p1)");
					zcc_errors.inc_error();
					memset(x.c_array()+pretokenized[i].first,' ',pretokenized[i].second);
					pretokenized.DeleteIdx(i);
					continue;
					}
				if (   C_TESTFLAG_IDENTIFIER!=pretokenized[i+1].third
					|| SIZE_MAX==lang.lex_find(arglist.data(),arglist.size(),x.data()+pretokenized[i+1].first,pretokenized[i+1].second))
					{	//! \test define.C99/Error_stringize3.hpp, define.C99/Error_stringize3.h
					message_header(x);
					INC_INFORM(ERR_STR);
					INFORM("# not followed by macro parameter; excising and continuing (C99 6.10.3.2p1/C++98 16.3.2p1)");
					zcc_errors.inc_error();
					memset(x.c_array()+pretokenized[i].first,' ',pretokenized[i].second);
					pretokenized.DeleteIdx(i);
					continue;
					}
				}
			++i;
			}

		if (detect_C_stringize_op(x.data()+pretokenized.back().first,pretokenized.back().second))
			{	//! \test define.C99/Error_stringize1.hpp, define.C99/Error_stringize1.h
			message_header(x);
			INC_INFORM(ERR_STR);
			INFORM("# terminates macro replacement list; truncating and continuing (C99 6.10.3.2p1/C++98 16.3.2p1)");
			zcc_errors.inc_error();
			if (1==pretokenized.size()) return x.reset(),true;
			pretokenized.DeleteIdx(pretokenized.size()-1);
			};
		remove_ws_from_token(x,pretokenized);
		}
	return false;
}

void CPreprocessor::object_macro_concatenate(Token<char>& x)
{	//! \pre: x is normalized
	//! \test cpp/default/Preprocess_macro_novar_concatenate.hpp, cpp/default/Preprocess_macro_novar_concatenate.h
	if (4>x.size()) return;
	assert(!strpbrk(x.data(),lang.WhiteSpace+2));	// check for normalization
	assert(!strchr(x.data(),'\n'));					// check for normalization
	autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> > pretokenized;
	const char* const x_data = x.data();
	if (lang.line_lex_find(x_data,x.size(),"##",sizeof("##")-1,pretokenized) || lang.line_lex_find(x_data,x.size(),"%:%:",sizeof("%:%:")-1,pretokenized))
		{
		assert(!pretokenized.empty());
		assert(!detect_C_concatenation_op(x_data+pretokenized.front().first,pretokenized.front().second));
		assert(!detect_C_concatenation_op(x_data+pretokenized.back().first, pretokenized.back().second));
		size_t i = pretokenized.size();
		do	{
			--i;
			if (detect_C_concatenation_op(x_data+pretokenized[i].first,pretokenized[i].second))
				{
				assert(0<i && pretokenized.size()-1>i);
				if (_concatenate_single(x,pretokenized.data()+(i-1),lang)) --i;
				}
			}
		while(0<i);
		}
}

void
CPreprocessor::function_macro_concatenate_novars(Token<char>& x, const Token<char>& arglist)
{	//! \pre: x is normalized
	//! \test cpp/default/Preprocess_macro_novar_concatenate.hpp, cpp/default/Preprocess_macro_novar_concatenate.h
	if (4>x.size()) return;
	assert(!strpbrk(x.data(),lang.WhiteSpace+2));	// check for normalization
	assert(!strchr(x.data(),'\n'));	// check for normalization
	const char* const x_data = x.data();
	autovalarray_ptr<POD_triple<size_t,size_t,lex_flags> > pretokenized;
	if (lang.line_lex_find(x_data,x.size(),"##",sizeof("##")-1,pretokenized) || lang.line_lex_find(x_data,x.size(),"%:%:",sizeof("%:%:")-1,pretokenized))
		{
		assert(!pretokenized.empty());
		assert(!detect_C_concatenation_op(x_data+pretokenized.front().first,pretokenized.front().second));
		assert(!detect_C_concatenation_op(x_data+pretokenized.back().first, pretokenized.back().second));
		size_t i = pretokenized.size();
		do	{
			--i;
			if (detect_C_concatenation_op(x_data+pretokenized[i].first,pretokenized[i].second))
				{
				assert(0<i && pretokenized.size()-1>i);
				const POD_triple<size_t,size_t,lex_flags>& tmp = pretokenized[i-1];
				const bool before_token_is_parameter = (C_TESTFLAG_IDENTIFIER==tmp.third) ? SIZE_MAX!=lang.lex_find(arglist.data(),arglist.size(),x_data+tmp.first,tmp.second) : false;

				if (detect_C_stringize_op(x_data+pretokenized[i+1].first,pretokenized[i+1].second))
					{
					message_header(x);
					INFORM("warning: order of evaluation of # and ## operators is undefined; evaluating # first (C99 6.10.3.2p2/C++98 16.3.2p2)");
					message_header(x);
					// would like these to be errors, but C99 requires accepting
					if (   !before_token_is_parameter						// not a parameter at all
						|| (1U<=tmp.first && '#'==x_data[tmp.first-1])
						|| (2U<=tmp.first && '%'==x_data[tmp.first-2] && ':'==x_data[tmp.first-1]))	// stringized parameter
						{	//! \test Warn_autofail_concatenation1.hpp
							//! \test default.nonconforming/Error_autofail_concatenation1.hpp
							//! \test Warn_autofail_concatenation2.hpp
							//! \test default.nonconforming/Error_autofail_concatenation2.hpp
						INC_INFORM((bool_options[boolopt::pedantic]) ? WARN_STR : ERR_STR);
						INC_INFORM("concatenation automatically fails at macro invocation.");
						}
					else{	//! \test Warn_empty_parameter_concatenation1.hpp
							//! \test default.nonconforming/Error_empty_parameter_concatenation1.hpp
						INC_INFORM((bool_options[boolopt::pedantic]) ? WARN_STR : ERR_STR);
						INC_INFORM("concatenation fails at macro invocation if parameter ");
						INC_INFORM(x_data+tmp.first,tmp.second);
						INC_INFORM(" is not empty.");
						}
					INFORM((bool_options[boolopt::pedantic]) ? "" : " (ZCPP nonconforming pragmatism)");
					// defer to here so we get the full messages out
					if (bool_options[boolopt::warnings_are_errors]) zcc_errors.inc_error();										// for order-of-evaluation
					if (bool_options[boolopt::warnings_are_errors] || !bool_options[boolopt::pedantic]) zcc_errors.inc_error();	// for wonky concatenation
					continue;
					}

				// will not be be able to complete concatenation against a parameter, bail
				// accept some inefficiency in a weird case to avoid code duplication
				const bool after_token_is_parameter = (C_TESTFLAG_IDENTIFIER==pretokenized[i+1].third) ? SIZE_MAX!=lang.lex_find(arglist.data(),arglist.size(),x_data+pretokenized[i+1].first,pretokenized[i+1].second) : false;

				// Of course, C/C++ can't concatenate a string with anything except an empty parameter.
				if (before_token_is_parameter)
					{
					if (	(1U<=tmp.first && '#'==x_data[tmp.first-1])
						|| 	(2U<=tmp.first && '%'==x_data[tmp.first-2] && ':'==x_data[tmp.first-1]))
						{	// stringized parameter
						message_header(x);
						INFORM("warning: order of evaluation of # and ## operators is undefined; evaluating # first (C99 6.10.3.2p2/C++98 16.3.2p2)");
						message_header(x);
						INC_INFORM((bool_options[boolopt::pedantic]) ? WARN_STR : ERR_STR);
						if (after_token_is_parameter)
							{	//! \test Warn_autofail_concatenation3.hpp
								//! \test default.nonconforming/Error_autofail_concatenation3.hpp
							INC_INFORM("concatenation fails at macro invocation if parameter ");
							INC_INFORM(x_data+pretokenized[i+1].first,pretokenized[i+1].second);
							INC_INFORM(" is not empty.");
							}
						else	//! \test Warn_empty_parameter_concatenation2.hpp
								//! \test default.nonconforming/Error_empty_parameter_concatenation2.hpp
							INC_INFORM("concatenation automatically fails at macro invocation.");
						INFORM((bool_options[boolopt::pedantic]) ? "" : " (ZCPP nonconforming pragmatism)");
						// defer to here so we get the full messages out
						if (bool_options[boolopt::warnings_are_errors]) zcc_errors.inc_error();										// for order-of-evaluation
						if (bool_options[boolopt::warnings_are_errors] || !bool_options[boolopt::pedantic]) zcc_errors.inc_error();	// for wonky concatenation
						};
					// will not be be able to complete concatenation against a parameter, bail
					continue;
					}

				// will not be be able to complete concatenation against a parameter, bail
				if (after_token_is_parameter) continue;

				if (_concatenate_single(x,pretokenized.data()+(i-1),lang)) --i;
				}
			}
		while(0<i);
		}
}

/*
 * analyzing constant expressions that can actually make it to a control expression (we have to be self-contained)
 ** no identifiers make it
 ** possible primary constants
 *** constant
 *** string-literal (must allow for token-combination)
 *** (expression)
 ** no post-fix operators are valid: [] () . -> postfix ++ postfix --
 *** well...since a string-literal is a char array [] would be valid on a string.... (and then we have to deal with code points)
 ** allowed unary operators: + - ~ ! (note that required integral type is one of the target's intmax_t or uintmax_t); this includes interpreting character constants.
 *** disallowed: prefix ++ prefix -- & * sizeof
 *** sizeof won't survive identifier zeroing
 *** * on a string-literal should dereference to a character, so allow &* cancellation on string literals
 *** allow *& cancellation always?
 ** no cast operators allowed
 ** allowed multiplicative operators: * / %
 ** allowed additive operators: + -
 ** allowed bitwise shift operators: << >>
 ** allowed relational operators: < > <= >=
 ** allowed equality operators: == !=
 ** bitwise and operator: &
 ** bitwise exclusive or: ^
 ** bitwise inclusive or: |
 ** logical and: &&
 ** logical or: ||
 ** conditional operator: ? :
 ** no assignment operators allowed
 ** comma operator: , (although probably not incredibly useful...)

 Since we need intmax_t and uintmax_t anyway to do this right: perhaps limits.h should be a virtual header generated from a target_info object
 automatic header also could handle the "no keywords defined when including a system header", but there probably is a more elegant way to do this

 Also: C99 7.1.2p4 prohibits all keywords from having macro definitions at the time of including the standard headers
<assert.h>
<complex.h>
<ctype.h>
<errno.h>
<fenv.h>
<float.h>
<inttypes.h>
<iso646.h>
<limits.h>
<locale.h>
<math.h>
<setjmp.h>
<signal.h>
<stdarg.h>
<stdbool.h>
<stddef.h>
<stdint.h>
<stdio.h>
<stdlib.h>
<string.h>
<tgmath.h>
<time.h>
<wchar.h>
<wctype.h>

C++98 17.3.3.1.1p2 goes further and prohibits defining macros for any names declared or defined in a library header.
<algorithm> <iomanip> <list> <ostream> <streambuf>
<bitset> <ios> <locale> <queue> <string>
<complex> <iosfwd> <map> <set> <typeinfo>
<deque> <iostream> <memory> <sstream> <utility>
<exception> <istream> <new> <stack> <valarray>
<fstream> <iterator> <numeric> <stdexcept> <vector>
<functional> <limits>

<cassert> <ciso646> <csetjmp> <cstdio> <ctime>
<cctype> <climits> <csignal> <cstdlib> <cwchar>
<cerrno> <clocale> <cstdarg> <cstring> <cwctype>
<cfloat> <cmath> <cstddef>

C++98 17.3.3.1.1p1 prohibits undefining any macro defined in a library header (watch out for assert.h)
C99 7.1.3p1,3 prohibit defining macros for any identifier declared in a library header
 */

