Add an mwt misc field to the wire format, including the protobuf annotation serializer

Uses a single string instead of a map for mwtMisc.  This is not ideal,
because it would be nice to search through the individual pieces, but
different treebanks use different standards for how to separate the
pieces, so we just follow the same convention as for words for now and
keep the whole thing as one string.

Author: AngledLuffa

src/edu/stanford/nlp/ling/AnnotationLookup.java

@@ -98,7 +98,8 @@ private enum KeyLookup {
     // MWT specific annotations
     MWT_TEXT_KEY(CoreAnnotations.MWTTokenTextAnnotation.class, "mwt_text"),
     IS_MWT_KEY(CoreAnnotations.IsMultiWordTokenAnnotation.class, "is_mwt"),
-    IS_FIRST_MWT_KEY(CoreAnnotations.IsFirstWordOfMWTAnnotation.class, "is_first_mwt");
+    IS_FIRST_MWT_KEY(CoreAnnotations.IsFirstWordOfMWTAnnotation.class, "is_first_mwt"),
+    MWT_MISC_KEY(CoreAnnotations.MWTTokenMiscAnnotation.class, "mwt_misc");
 
     private final Class<? extends CoreAnnotation<?>> coreKey;
     private final String oldKey;

src/edu/stanford/nlp/ling/CoreAnnotations.java

@@ -2217,5 +2217,13 @@ public Class<Boolean> getType() {
     }
   }
 
-
+  /**
+   * CoNLL-U misc features specifically on the MWT part of a token rather than the word
+   */
+  public static class MWTTokenMiscAnnotation implements CoreAnnotation<String> {
+    @Override
+    public Class<String> getType() {
+      return String.class;
+    }
+  }
 }

src/edu/stanford/nlp/pipeline/CoreNLP.proto

@@ -222,6 +222,11 @@ message Token {
   optional bool isMWT = 67;
   optional bool isFirstMWT = 68;
   optional string mwtText = 69;
+  // setting this to a map might be nice, but there are a couple issues
+  // for one, there can be values with no key
+  // for another, it's a pain to correctly parse, since different treebanks
+  // can have different standards for how to write out the misc field
+  optional string mwtMisc = 78;
 
   // number info
   optional uint64 numericValue = 70;