Update symbol generation scripts

Summary:
Scripts now return scoped symbols -- in particular, PHP class constants, fields, and methods. ctags gives some for other languages.

(Turns out XHPAST doesn't support traits. But no one uses traits anyway so it's probably fine.)

I couldn't find a list of the context types ctags uses (class/struct/union/enum/maybe others?), so the context code just ignores that. Also, it uses a blacklist for the symbol type instead of a whitelist because there are a ton, they vary by language, and I didn't want to unintentionally exclude anything (P480).

Test Plan: Scrape symbols from arcanist and phabricator. Upload them to sandbox. Search for things.

Reviewers: epriestley

Reviewed By: epriestley

CC: nh, aran, Korvin

Maniphest Tasks: T1602

Differential Revision: https://secure.phabricator.com/D3202

scripts/symbols/generate_php_symbols.php

@@ -45,6 +45,7 @@ foreach (Futures($futures)->limit(8) as $file => $future) {
     $future->resolve());
 
   $root = $tree->getRootNode();
+  $scopes = array();
 
   $functions = $root->selectDescendantsOfType('n_FUNCTION_DECLARATION');
   foreach ($functions as $function) {
@@ -56,18 +57,64 @@ foreach (Futures($futures)->limit(8) as $file => $future) {
   foreach ($classes as $class) {
     $class_name = $class->getChildByIndex(1);
     print_symbol($file, 'class', $class_name);
+    $scopes[] = array($class, $class_name);
   }
 
   $interfaces = $root->selectDescendantsOfType('n_INTERFACE_DECLARATION');
   foreach ($interfaces as $interface) {
     $interface_name = $interface->getChildByIndex(1);
     print_symbol($file, 'interface', $interface_name);
+    $scopes[] = array($interface, $interface_name);
+  }
+
+  $constants = $root->selectDescendantsOfType('n_CONSTANT_DECLARATION_LIST');
+  foreach ($constants as $constant_list) {
+    foreach ($constant_list->getChildren() as $constant) {
+      $constant_name = $constant->getChildByIndex(0);
+      print_symbol($file, 'constant', $constant_name);
+    }
+  }
+
+  foreach ($scopes as $scope) {
+    // this prints duplicate symbols in the case of nested classes
+    // luckily, PHP doesn't allow those
+    list($class, $class_name) = $scope;
+
+    $consts = $class->selectDescendantsOfType(
+      'n_CLASS_CONSTANT_DECLARATION_LIST');
+    foreach ($consts as $const_list) {
+      foreach ($const_list->getChildren() as $const) {
+        $const_name = $const->getChildByIndex(0);
+        print_symbol($file, 'class_const', $const_name, $class_name);
+      }
+    }
+
+    $members = $class->selectDescendantsOfType(
+      'n_CLASS_MEMBER_DECLARATION_LIST');
+    foreach ($members as $member_list) {
+      foreach ($member_list->getChildren() as $member) {
+        if ($member->getTypeName() == 'n_CLASS_MEMBER_MODIFIER_LIST') {
+          continue;
+        }
+        $member_name = $member->getChildByIndex(0);
+        print_symbol($file, 'member', $member_name, $class_name);
+      }
+    }
+
+    $methods = $class->selectDescendantsOfType('n_METHOD_DECLARATION');
+    foreach ($methods as $method) {
+      $method_name = $method->getChildByIndex(2);
+      print_symbol($file, 'method', $method_name, $class_name);
+    }
   }
 }
 
-function print_symbol($file, $type, $token) {
+function print_symbol($file, $type, $token, $context=null) {
   $parts = array(
-    $token->getConcreteString(),
+    $context ? $context->getConcreteString() : '',
+    // variable tokens are `$name`, not just `name`, so strip the $ off of
+    // class field names
+    ltrim($token->getConcreteString(), '$'),
     $type,
     'php',
     $token->getLineNumber(),