rustc: Implement custom derive (macros 1.1)

This commit is an implementation of [RFC 1681] which adds support to the
compiler for first-class user-define custom `#[derive]` modes with a far more
stable API than plugins have today.

[RFC 1681]: https://github.com/rust-lang/rfcs/blob/master/text/1681-macros-1.1.md

The main features added by this commit are:

* A new `rustc-macro` crate-type. This crate type represents one which will
  provide custom `derive` implementations and perhaps eventually flower into the
  implementation of macros 2.0 as well.

* A new `rustc_macro` crate in the standard distribution. This crate will
  provide the runtime interface between macro crates and the compiler. The API
  here is particularly conservative right now but has quite a bit of room to
  expand into any manner of APIs required by macro authors.

* The ability to load new derive modes through the `#[macro_use]` annotations on
  other crates.

All support added here is gated behind the `rustc_macro` feature gate, both for
the library support (the `rustc_macro` crate) as well as the language features.

There are a few minor differences from the implementation outlined in the RFC,
such as the `rustc_macro` crate being available as a dylib and all symbols are
`dlsym`'d directly instead of having a shim compiled. These should only affect
the implementation, however, not the public interface.

This commit also ended up touching a lot of code related to `#[derive]`, making
a few notable changes:

* Recognized derive attributes are no longer desugared to `derive_Foo`. Wasn't
  sure how to keep this behavior and *not* expose it to custom derive.

* Derive attributes no longer have access to unstable features by default, they
  have to opt in on a granular level.

* The `derive(Copy,Clone)` optimization is now done through another "obscure
  attribute" which is just intended to ferry along in the compiler that such an
  optimization is possible. The `derive(PartialEq,Eq)` optimization was also
  updated to do something similar.

---

One part of this PR which needs to be improved before stabilizing are the errors
and exact interfaces here. The error messages are relatively poor quality and
there are surprising spects of this such as `#[derive(PartialEq, Eq, MyTrait)]`
not working by default. The custom attributes added by the compiler end up
becoming unstable again when going through a custom impl.

Hopefully though this is enough to start allowing experimentation on crates.io!

syntax-[breaking-change]

Author: alexcrichton

mk/crates.mk

@@ -59,7 +59,7 @@ RUSTC_CRATES := rustc rustc_typeck rustc_mir rustc_borrowck rustc_resolve rustc_
                 rustc_trans rustc_back rustc_llvm rustc_privacy rustc_lint \
                 rustc_data_structures rustc_platform_intrinsics rustc_errors \
                 rustc_plugin rustc_metadata rustc_passes rustc_save_analysis \
-                rustc_const_eval rustc_const_math rustc_incremental
+                rustc_const_eval rustc_const_math rustc_incremental rustc_macro
 HOST_CRATES := syntax syntax_ext proc_macro syntax_pos $(RUSTC_CRATES) rustdoc fmt_macros \
 		flate arena graphviz rbml log serialize
 TOOLS := compiletest rustdoc rustc rustbook error_index_generator
@@ -99,7 +99,7 @@ DEPS_term := std
 DEPS_test := std getopts term native:rust_test_helpers
 
 DEPS_syntax := std term serialize log arena libc rustc_bitflags rustc_unicode rustc_errors syntax_pos
-DEPS_syntax_ext := syntax syntax_pos rustc_errors fmt_macros
+DEPS_syntax_ext := syntax syntax_pos rustc_errors fmt_macros rustc_macro
 DEPS_proc_macro := syntax syntax_pos rustc_plugin log
 DEPS_syntax_pos := serialize
 
@@ -118,11 +118,13 @@ DEPS_rustc_driver := arena flate getopts graphviz libc rustc rustc_back rustc_bo
                      rustc_trans rustc_privacy rustc_lint rustc_plugin \
                      rustc_metadata syntax_ext proc_macro \
                      rustc_passes rustc_save_analysis rustc_const_eval \
-                     rustc_incremental syntax_pos rustc_errors
+                     rustc_incremental syntax_pos rustc_errors rustc_macro
 DEPS_rustc_errors := log libc serialize syntax_pos
 DEPS_rustc_lint := rustc log syntax syntax_pos rustc_const_eval
 DEPS_rustc_llvm := native:rustllvm libc std rustc_bitflags
-DEPS_rustc_metadata := rustc syntax syntax_pos rustc_errors rbml rustc_const_math
+DEPS_rustc_macro := std syntax
+DEPS_rustc_metadata := rustc syntax syntax_pos rustc_errors rbml rustc_const_math \
+			rustc_macro syntax_ext
 DEPS_rustc_passes := syntax syntax_pos rustc core rustc_const_eval rustc_errors
 DEPS_rustc_mir := rustc syntax syntax_pos rustc_const_math rustc_const_eval rustc_bitflags
 DEPS_rustc_resolve := arena rustc log syntax syntax_pos rustc_errors

src/librustc/middle/dependency_format.rs

@@ -139,8 +139,13 @@ fn calculate_type(sess: &session::Session,
             }
         }
 
-        // Everything else falls through below
-        config::CrateTypeExecutable | config::CrateTypeDylib => {},
+        // Everything else falls through below. This will happen either with the
+        // `-C prefer-dynamic` or because we're a rustc-macro crate. Note that
+        // rustc-macro crates are required to be dylibs, and they're currently
+        // required to link to libsyntax as well.
+        config::CrateTypeExecutable |
+        config::CrateTypeDylib |
+        config::CrateTypeRustcMacro => {},
     }
 
     let mut formats = FnvHashMap();

src/librustc/middle/reachable.rs

@@ -138,7 +138,8 @@ impl<'a, 'tcx> ReachableContext<'a, 'tcx> {
     // Creates a new reachability computation context.
     fn new(tcx: TyCtxt<'a, 'tcx, 'tcx>) -> ReachableContext<'a, 'tcx> {
         let any_library = tcx.sess.crate_types.borrow().iter().any(|ty| {
-            *ty == config::CrateTypeRlib || *ty == config::CrateTypeDylib
+            *ty == config::CrateTypeRlib || *ty == config::CrateTypeDylib ||
+            *ty == config::CrateTypeRustcMacro
         });
         ReachableContext {
             tcx: tcx,

src/librustc/middle/weak_lang_items.rs

@@ -70,6 +70,7 @@ fn verify(sess: &Session, items: &lang_items::LanguageItems) {
     let needs_check = sess.crate_types.borrow().iter().any(|kind| {
         match *kind {
             config::CrateTypeDylib |
+            config::CrateTypeRustcMacro |
             config::CrateTypeCdylib |
             config::CrateTypeExecutable |
             config::CrateTypeStaticlib => true,

src/librustc/session/config.rs

@@ -475,6 +475,7 @@ pub enum CrateType {
     CrateTypeRlib,
     CrateTypeStaticlib,
     CrateTypeCdylib,
+    CrateTypeRustcMacro,
 }
 
 #[derive(Clone, Hash)]
@@ -962,6 +963,9 @@ pub fn default_configuration(sess: &Session) -> ast::CrateConfig {
     if sess.opts.debug_assertions {
         ret.push(attr::mk_word_item(InternedString::new("debug_assertions")));
     }
+    if sess.opts.crate_types.contains(&CrateTypeRustcMacro) {
+        ret.push(attr::mk_word_item(InternedString::new("rustc_macro")));
+    }
     return ret;
 }
 
@@ -1547,6 +1551,7 @@ pub fn parse_crate_types_from_list(list_list: Vec<String>) -> Result<Vec<CrateTy
                 "dylib"     => CrateTypeDylib,
                 "cdylib"    => CrateTypeCdylib,
                 "bin"       => CrateTypeExecutable,
+                "rustc-macro" => CrateTypeRustcMacro,
                 _ => {
                     return Err(format!("unknown crate type: `{}`",
                                        part));
@@ -1635,6 +1640,7 @@ impl fmt::Display for CrateType {
             CrateTypeRlib => "rlib".fmt(f),
             CrateTypeStaticlib => "staticlib".fmt(f),
             CrateTypeCdylib => "cdylib".fmt(f),
+            CrateTypeRustcMacro => "rustc-macro".fmt(f),
         }
     }
 }

src/librustc/session/mod.rs

@@ -62,6 +62,7 @@ pub struct Session {
     pub entry_fn: RefCell<Option<(NodeId, Span)>>,
     pub entry_type: Cell<Option<config::EntryFnType>>,
     pub plugin_registrar_fn: Cell<Option<ast::NodeId>>,
+    pub derive_registrar_fn: Cell<Option<ast::NodeId>>,
     pub default_sysroot: Option<PathBuf>,
     // The name of the root source file of the crate, in the local file system.
     // The path is always expected to be absolute. `None` means that there is no
@@ -314,6 +315,12 @@ impl Session {
         format!("__rustc_plugin_registrar__{}_{}", svh, index.as_usize())
     }
 
+    pub fn generate_derive_registrar_symbol(&self,
+                                            svh: &Svh,
+                                            index: DefIndex) -> String {
+        format!("__rustc_derive_registrar__{}_{}", svh, index.as_usize())
+    }
+
     pub fn sysroot<'a>(&'a self) -> &'a Path {
         match self.opts.maybe_sysroot {
             Some (ref sysroot) => sysroot,
@@ -501,6 +508,7 @@ pub fn build_session_(sopts: config::Options,
         entry_fn: RefCell::new(None),
         entry_type: Cell::new(None),
         plugin_registrar_fn: Cell::new(None),
+        derive_registrar_fn: Cell::new(None),
         default_sysroot: default_sysroot,
         local_crate_source_file: local_crate_source_file,
         working_dir: env::current_dir().unwrap(),

src/librustc/ty/context.rs

@@ -495,6 +495,10 @@ pub struct GlobalCtxt<'tcx> {
 
     /// Cache for layouts computed from types.
     pub layout_cache: RefCell<FnvHashMap<Ty<'tcx>, &'tcx Layout>>,
+
+    /// Map from function to the `#[derive]` mode that it's defining. Only used
+    /// by `rustc-macro` crates.
+    pub derive_macros: RefCell<NodeMap<token::InternedString>>,
 }
 
 impl<'tcx> GlobalCtxt<'tcx> {
@@ -756,6 +760,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
             crate_name: token::intern_and_get_ident(crate_name),
             data_layout: data_layout,
             layout_cache: RefCell::new(FnvHashMap()),
+            derive_macros: RefCell::new(NodeMap()),
        }, f)
     }
 }

src/librustc_driver/derive_registrar.rs

@@ -0,0 +1,37 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use rustc::dep_graph::DepNode;
+use rustc::hir::intravisit::Visitor;
+use rustc::hir::map::Map;
+use rustc::hir;
+use syntax::ast;
+use syntax::attr;
+
+pub fn find(hir_map: &Map) -> Option<ast::NodeId> {
+    let _task = hir_map.dep_graph.in_task(DepNode::PluginRegistrar);
+    let krate = hir_map.krate();
+
+    let mut finder = Finder { registrar: None };
+    krate.visit_all_items(&mut finder);
+    finder.registrar
+}
+
+struct Finder {
+    registrar: Option<ast::NodeId>,
+}
+
+impl<'v> Visitor<'v> for Finder {
+    fn visit_item(&mut self, item: &hir::Item) {
+        if attr::contains_name(&item.attrs, "rustc_derive_registrar") {
+            self.registrar = Some(item.id);
+        }
+    }
+}

src/librustc_driver/driver.rs

@@ -55,6 +55,8 @@ use syntax::util::node_count::NodeCounter;
 use syntax;
 use syntax_ext;
 
+use derive_registrar;
+
 #[derive(Clone)]
 pub struct Resolutions {
     pub def_map: DefMap,
@@ -696,6 +698,18 @@ pub fn phase_2_configure_and_expand<'a, F>(sess: &Session,
                                          sess.diagnostic())
     });
 
+    krate = time(time_passes, "maybe creating a macro crate", || {
+        let crate_types = sess.crate_types.borrow();
+        let is_rustc_macro_crate = crate_types.contains(&config::CrateTypeRustcMacro);
+        let num_crate_types = crate_types.len();
+        syntax_ext::rustc_macro_registrar::modify(&sess.parse_sess,
+                                                  krate,
+                                                  is_rustc_macro_crate,
+                                                  num_crate_types,
+                                                  sess.diagnostic(),
+                                                  &sess.features.borrow())
+    });
+
     let resolver_arenas = Resolver::arenas();
     let mut resolver = Resolver::new(sess, make_glob_map, &resolver_arenas);
 
@@ -838,6 +852,7 @@ pub fn phase_3_run_analysis_passes<'tcx, F, R>(sess: &'tcx Session,
     sess.plugin_registrar_fn.set(time(time_passes, "looking for plugin registrar", || {
         plugin::build::find_plugin_registrar(sess.diagnostic(), &hir_map)
     }));
+    sess.derive_registrar_fn.set(derive_registrar::find(&hir_map));
 
     let region_map = time(time_passes,
                           "region resolution",
@@ -1171,6 +1186,9 @@ pub fn collect_crate_types(session: &Session, attrs: &[ast::Attribute]) -> Vec<c
                          Some(ref n) if *n == "staticlib" => {
                              Some(config::CrateTypeStaticlib)
                          }
+                         Some(ref n) if *n == "rustc-macro" => {
+                             Some(config::CrateTypeRustcMacro)
+                         }
                          Some(ref n) if *n == "bin" => Some(config::CrateTypeExecutable),
                          Some(_) => {
                              session.add_lint(lint::builtin::UNKNOWN_CRATE_TYPES,

src/librustc_driver/lib.rs

@@ -107,7 +107,7 @@ pub mod test;
 pub mod driver;
 pub mod pretty;
 pub mod target_features;
-
+mod derive_registrar;
 
 const BUG_REPORT_URL: &'static str = "https://github.com/rust-lang/rust/blob/master/CONTRIBUTING.\
                                       md#bug-reports";

src/librustc_macro/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "rustc_macro"
+version = "0.0.0"
+
+[lib]
+name = "rustc_macro"
+path = "lib.rs"
+crate-type = ["dylib"]
+
+[dependencies]
+syntax = { path = "../libsyntax" }