gcc 源码阅读--C语言预处理

在c/c++语言中，编译器第一项处理的就是做预处理，

比如#include ,#ifdef #endif  #if 等等

GCC实现这部分代码放在一个单独的目录中,libcpp

这里面有几个重要结构需要了解：

struct GTY(()) cpp_token {

  /* Location of first char of token, together with range of full token.  */
location_t src_loc;
// 记录词法元素中第一个字符的源码位置 

ENUM_BITFIELD(cpp_ttype) type : CHAR_BIT;  /* token type */
//对应的类型，CPP_NUMBER
unsigned short flags;     /* flags - see above */

  union cpp_token_u
{
// 代表的是为各个词法元素最终建立的值节点，不同的词法元素使用不同的结构体。
/* An identifier.  */
struct cpp_identifier GTY ((tag ("CPP_TOKEN_FLD_NODE"))) node;

    /* Inherit padding from this token.  */
cpp_token * GTY ((tag ("CPP_TOKEN_FLD_SOURCE"))) source;

    /* A string, or number.  */
struct cpp_string GTY ((tag ("CPP_TOKEN_FLD_STR"))) str;

    /* Argument no. (and original spelling) for a CPP_MACRO_ARG.  */
struct cpp_macro_arg GTY ((tag ("CPP_TOKEN_FLD_ARG_NO"))) macro_arg;

    /* Original token no. for a CPP_PASTE (from a sequence of
consecutive paste tokens in a macro expansion).  */
unsigned int GTY ((tag ("CPP_TOKEN_FLD_TOKEN_NO"))) token_no;

    /* Caller-supplied identifier for a CPP_PRAGMA.  */
unsigned int GTY ((tag ("CPP_TOKEN_FLD_PRAGMA"))) pragma;
} GTY ((desc ("cpp_token_val_index (&%1)"))) val;
};
struct GTY (()) c_token {
/* The kind of token.  */
ENUM_BITFIELD (cpp_ttype) type : 8;
/* If this token is a CPP_NAME, this value indicates whether also
declared as some kind of type.  Otherwise, it is C_ID_NONE.  */
ENUM_BITFIELD (c_id_kind) id_kind : 8;//关键字标识符
/* If this token is a keyword, this value indicates which keyword.
Otherwise, this value is RID_MAX.  */
ENUM_BITFIELD (rid) keyword : 8;
/* If this token is a CPP_PRAGMA, this indicates the pragma that
was seen.  Otherwise it is PRAGMA_NONE.  */
ENUM_BITFIELD (pragma_kind) pragma_kind : 8;
/* The location at which this token was found.  */
location_t location;
/* The value associated with this token, if any.  */
tree value;
/* Token flags.  */
unsigned char flags;

  source_range get_range () const
{
return get_range_from_loc (line_table, location);
}

  location_t get_finish () const
{
return get_range ().m_finish;
}
};

struct c_parser {

  c_token * tokens;    /* 当前正在处理的语法符号c_token的地址，这里除了初始化时，应该指向 tokens_buf[0] */

  c_token tokens_buf[4];  /* c_token预读缓存，按照gcc的语法分析原理，预读不会超过4个语法符号 */

  unsigned int tokens_avail;    /* tokens_buf中可用的预读词法符号的数目 */
。。。。。。
}

struct cpp_reader
{
/* Top of buffer stack.  */
cpp_buffer *buffer;

  /* Overlaid buffer (can be different after processing #include).  */
cpp_buffer *overlaid_buffer;

  /* Lexer state.  */
struct lexer_state state;

  /* Source line tracking.  */
class line_maps *line_table;

  /* The line of the '#' of the current directive.  */
location_t directive_line;

  /* Memory buffers.  */
_cpp_buff *a_buff;        /* Aligned permanent storage.  */
_cpp_buff *u_buff;        /* Unaligned permanent storage.  */
_cpp_buff *free_buffs;    /* Free buffer chain.  */

  /* Context stack.  */
struct cpp_context base_context;
struct cpp_context *context;

  /* If in_directive, the directive if known.  */
const struct directive *directive;

  /* Token generated while handling a directive, if any. */
cpp_token directive_result;

  /* When expanding a macro at top-level, this is the location of the
macro invocation.  */
location_t invocation_location;

  /* This is the node representing the macro being expanded at
top-level.  The value of this data member is valid iff
cpp_in_macro_expansion_p() returns TRUE.  */
cpp_hashnode *top_most_macro_node;

  /* Nonzero if we are about to expand a macro.  Note that if we are
really expanding a macro, the function macro_of_context returns
the macro being expanded and this flag is set to false.  Client
code should use the function cpp_in_macro_expansion_p to know if we
are either about to expand a macro, or are actually expanding
one.  */
bool about_to_expand_macro_p;

  /* True if the preprocessor should diagnose CPP_DOT or CPP_COLON
tokens as the first ones coming from macro expansion.  */
bool diagnose_dot_colon_from_macro_p;

  /* Search paths for include files.  */
struct cpp_dir *quote_include;    /* "" */
struct cpp_dir *bracket_include;    /* <> */
struct cpp_dir no_search_path;    /* No path.  */
struct cpp_dir *embed_include;    /* #embed <> */

  /* Chain of all hashed _cpp_file instances.  */
struct _cpp_file *all_files;

  struct _cpp_file *main_file;

  /* File and directory hash table.  */
struct htab *file_hash;
struct htab *dir_hash;
struct file_hash_entry_pool *file_hash_entries;

  /* Negative path lookup hash table.  */
struct htab *nonexistent_file_hash;
struct obstack nonexistent_file_ob;

  /* Nonzero means don't look for #include "foo" the source-file
directory.  */
bool quote_ignores_source_dir;

  /* Nonzero if any file has contained #pragma once or #import has
been used.  */
bool seen_once_only;

  /* Multiple include optimization and -Wheader-guard warning.  */
const cpp_hashnode *mi_cmacro;
const cpp_hashnode *mi_ind_cmacro;
const cpp_hashnode *mi_def_cmacro;
location_t mi_loc, mi_def_loc;
bool mi_valid;

  /* Lexing.  */
cpp_token *cur_token;
tokenrun base_run, *cur_run;
unsigned int lookaheads;

  /* Nonzero prevents the lexer from re-using the token runs.  */
unsigned int keep_tokens;

  /* Buffer to hold macro definition string.  */
unsigned char *macro_buffer;
unsigned int macro_buffer_len;

  /* Descriptor for converting from the source character set to the
execution character set.  */
struct cset_converter narrow_cset_desc;

  /* Descriptor for converting from the source character set to the
UTF-8 execution character set.  */
struct cset_converter utf8_cset_desc;

  /* Descriptor for converting from the source character set to the
UTF-16 execution character set.  */
struct cset_converter char16_cset_desc;

  /* Descriptor for converting from the source character set to the
UTF-32 execution character set.  */
struct cset_converter char32_cset_desc;

  /* Descriptor for converting from the source character set to the
wide execution character set.  */
struct cset_converter wide_cset_desc;

  /* Date and time text.  Calculated together if either is requested.  */
const unsigned char *date;
const unsigned char *time;

  /* Time stamp, set idempotently lazily.  */
time_t time_stamp;
int time_stamp_kind; /* Or errno.  */

  /* A token forcing paste avoidance, and one demarking macro arguments.  */
cpp_token avoid_paste;
cpp_token endarg;

  /* Opaque handle to the dependencies of mkdeps.cc.  */
class mkdeps *deps;

  /* Obstack holding all macro hash nodes.  This never shrinks.
See identifiers.cc */
struct obstack hash_ob;

  /* Obstack holding buffer and conditional structures.  This is a
real stack.  See directives.cc.  */
struct obstack buffer_ob;

  /* Pragma table - dynamic, because a library user can add to the
list of recognized pragmas.  */
struct pragma_entry *pragmas;

  /* Call backs to cpplib client.  */
struct cpp_callbacks cb;

  /* Identifier hash table.  */
struct ht *hash_table;

  /* Identifier ancillary data hash table.  */
struct ht *extra_hash_table;

  /* Expression parser stack.  */
struct op *op_stack, *op_limit;

  /* User visible options.  */
struct cpp_options opts;

  /* Special nodes - identifiers with predefined significance to the
preprocessor.  */
struct spec_nodes spec_nodes;

  /* Whether cpplib owns the hashtable.  */
bool our_hashtable, our_extra_hashtable;

  /* Traditional preprocessing output buffer (a logical line).  */
struct
{
unsigned char *base;
unsigned char *limit;
unsigned char *cur;
location_t first_line;
} out;

  /* Used for buffer overlays by traditional.cc.  */
const unsigned char *saved_cur, *saved_rlimit, *saved_line_base;

  /* A saved list of the defined macros, for dependency checking
of precompiled headers.  */
struct cpp_savedstate *savedstate;

  /* Next value of __COUNTER__ macro. */
unsigned int counter;

  /* Table of comments, when state.save_comments is true.  */
cpp_comment_table comments;

  /* List of saved macros by push_macro.  */
struct def_pragma_macro *pushed_macros;

  /* If non-zero, the lexer will use this location for the next token
instead of getting a location from the linemap.  */
location_t forced_token_location;

  /* Location identifying the main source file -- intended to be line
zero of said file.  */
location_t main_loc;

  /* If non-zero, override diagnostic locations (other than DK_NOTE
diagnostics) to this one.  */
location_t diagnostic_override_loc;

  /* Returns true iff we should warn about UTF-8 bidirectional control
characters.  */
bool warn_bidi_p () const
{
return (CPP_OPTION (this, cpp_warn_bidirectional)
& (bidirectional_unpaired|bidirectional_any));
}
}

/* An identifier hash table for cpplib and the front ends.  */
struct ht
{
/* Identifiers are allocated from here.  */
struct obstack stack;  //负责此hash表中的内存分配

   /*指向一个hashnode[nslots]数组的首地址，这个数组就是所谓的hash桶，数组中的每个元素都记录了一个具体元素的指针(所以每个元素叫做一个hashnode)
而hashnode具体的元素则是一个 ht_identifer,其只能代表一个字符串的内容，长度和hash.
此hash桶是自动扩展的，在ht搜索函数ht_lookup_with_hash中，若发现整个hash table超过3/4都满了，就会主动扩展此hash table(重新分配，复制，释放原有的)。*/

  hashnode *entries;
/* Call back, allocate a node.  */

/*
整个gcc源码中有两个alloc_node函数，一个定义在./gcc/stringpool.c中，一个定义在libcpp/identifiers.c中(libcpp这个目录是负责预处理和词法分析的).
* 对于cc1来说，其有自己的alloc_node函数，调用的总是 gcc/stringpools.c:alloc_node
* 而对于使用libcpp.a的其他程序，如果自己没有实现alloc_node函数，那么会默认使用./libcpp/identifier.c:alloc_node函数
alloc_node函数是用来分配节点内存的，分配后hashnode[]数组中的指针，也就指向这个内存中的元素，在ht搜索过程中(ht_lookup_with_hash),若发现需要新
插入一个元素，则就会调用alloc_node来分配内存，最终其返回值会被记录到hashnode[]中。
注: alloc_node可以为节点分配任意类型的结构体，只要最终返回此结构体中的一个ht_identifer结构体即可(./gcc/stringpool.c真正分配的是一个 lang_identifier树节点
*/
hashnode (*alloc_node) (cpp_hash_table *);
/* Call back, allocate something that hangs off a node like a cpp_macro.  
NULL means use the usual allocator.  */
void * (*alloc_subobject) (size_t);

  unsigned int nslots;      // hash中总共能存多少个指针
unsigned int nelements;   /* Number of live elements.  */

  /* Link to reader, if any.  For the benefit of cpplib.  */
struct cpp_reader *pfile;  //指向对应的cpp_reader(即parse_in)

  /* Table usage statistics.  */
unsigned int searches;
unsigned int collisions;

  /* Should 'entries' be freed when it is no longer needed?  */
bool entries_owned;
};

实现代码在：

toplev::main 
=>lang_hooks.init_options = c_common_init_options
=> parse_in = cpp_create_reader()                         //1) 这里主要是对全局变量parse_in的初始化
=> do_compile
=> process_options
=> lang_hooks.post_options = c_common_post_options
=> cpp_read_main_file (parse_in, in_fnames[0])        //2) 这里主要负责打开并读入编译单元文件
=> compile_file();
=> lang_hooks.parse_file = c_common_parse_file()
=> c_parse_file ();  

在这个函数中会初始化模块需要用到的几个参数：

init_stringpool (void)
{
/* Clean up if we're called more than once.
(We can't make this idempotent since identifiers contain state) */
if (ident_hash)
ht_destroy (ident_hash);
if (ident_hash_extra)
ht_destroy (ident_hash_extra);

  /* Create with 16K (2^14) entries.  */
ident_hash = ht_create (14);
ident_hash->alloc_node = alloc_node;
ident_hash->alloc_subobject = stringpool_ggc_alloc;

  /* Create with 64 (2^6) entries.  */
ident_hash_extra = ht_create (6);
ident_hash_extra->alloc_node = [] (cpp_hash_table *)
{
return HT_NODE (ggc_cleared_alloc<cpp_hashnode_extra> ());
};
ident_hash_extra->alloc_subobject = stringpool_ggc_alloc;
}
general_init (const char *argv0, bool init_signals, unique_argv original_argv)

{

.....

line_table = ggc_alloc<line_maps> ();
linemap_init (line_table, BUILTINS_LOCATION);
line_table->m_reallocator = realloc_for_line_map;
line_table->m_round_alloc_size = ggc_round_alloc_size;
line_table->default_range_bits = line_map_suggested_range_bits;

......

}

c_common_init_options (unsigned int decoded_options_count,
struct cl_decoded_option *decoded_options)
{
unsigned int i;
struct cpp_callbacks *cb;

  g_string_concat_db
= new (ggc_alloc <string_concat_db> ()) string_concat_db ();

  parse_in = cpp_create_reader (c_dialect_cxx () ? CLK_GNUCXX : CLK_GNUC89,
ident_hash, line_table, ident_hash_extra);
cb = cpp_get_callbacks (parse_in);
cb->diagnostic = c_cpp_diagnostic;

.....

}

/* Post-switch processing.  */
bool
c_common_post_options (const char **pfilename)  

{

...

 *pfilename = this_input_filename
= cpp_read_main_file (parse_in, in_fnames[0],
/* We'll inject preamble pieces if this is
not preprocessed.  */
!cpp_opts->preprocessed);

...

}

c_common_parse_file (void)
{
auto dumps = g->get_dumps ();
for (unsigned int i = 0;;)
{
c_finish_options ();
/* Open the dump file to use for the original dump output
here, to be used during parsing for the current file.  */
dumps->dump_start (TDI_original, &dump_flags);
pch_init ();
push_file_scope ();
c_parse_file ();
pop_file_scope ();
/* And end the main input file, if the debug writer wants it  */
if (debug_hooks->start_end_main_source_file)
(*debug_hooks->end_source_file) (0);
if (++i >= num_in_fnames)
break;
cpp_undef_all (parse_in);
cpp_clear_file_cache (parse_in);
this_input_filename
= cpp_read_main_file (parse_in, in_fnames[i]);
/* If an input file is missing, abandon further compilation.
cpplib has issued a diagnostic.  */
if (!this_input_filename)
break;
dumps->dump_finish (TDI_original);
}

  c_parse_final_cleanups ();
dumps->dump_finish (TDI_original);
}

/* Wrapper around cpp_get_token to skip CPP_PADDING tokens
and not consume CPP_EOF.  This does not perform the optional
streaming in preprocess_only mode, so is suitable to be used
when processing builtin expansions such as c_common_has_attribute.  */

static const cpp_token *
get_token_no_padding (cpp_reader *pfile)
{
for (;;)
{
const cpp_token *ret = cpp_peek_token (pfile, 0);
if (ret->type == CPP_EOF)
return ret;
ret = cpp_get_token (pfile);
if (ret->type != CPP_PADDING)
return ret;
}
}