archive/phabricator
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Move Diviner further toward usability

Browse Source 
Summary:
  - Complete the "project" -> "book" stuff. This is cleaner conceptually and keeps us from having yet another meaning for the word "project".
  - Normalize symbols during atomization. This simplifies publishing a great deal, and allows static documentation to link to dynamic documentation and vice versa, because the canonical names of symbols are agreed upon (we can tweak the actual algorithm).
  - Give articles a specifiable name distinct from the title, and default to something like "support" instead of "Get Help! Get Support!" so URIs end up more readable (not "Get_Help!_Get_Support!").
  - Have the atomizers set book information on atoms.
  - Implement very basic publishers. Publishers are basically glue code between the atomization process and the rendering process -- the two we'll have initially are "static" (publish to files on disk) and "phabricator" (or similar -- publish into the database).
  - Handle duplicate symbol definitions in the atomize and publish pipelines. This fixes the issue where a project defines two functions named "idx()" and we currently tell them not to do that and break. Realistically, this is common in the real world and we should just roll our eyes and do the legwork to generate documentation as best we can.
  - Particularly, dirty all atoms with the same name as a dirty atom (e.g., if 'function f()' is updated, regnerate the documentation for all functions named f() in the book).
  - When publishing, we publish these at "function/f/@1", "function/f/@2". The base page will offer to disambiguate ("There are 8 functions named 'f' in this codebase, which one do you want?").
  - Implement a very very basic renderer. This generates the actual HTML (or text, or XML, or whatever else) for the documentation, which the publisher dumps onto disk or into a database or whatever.
  - The atomize workflow actually needs to depend on books, at least sort of, so make it load config and use it properly.
  - Propagate multilevel dirties through the graph. If "C extends B" and "B extends A", we should regenerate C when A changes. Prior to this diff, we would regnerate B only.

Test Plan: Generated some documentation. Named two articles "feedback", generated docs, saw "article/feedback/@1/" and "article/feedback/@2/" created.

Reviewers: btrahan, vrana, chad

Reviewed By: chad

CC: aran

Maniphest Tasks: T988

Differential Revision: https://secure.phabricator.com/D4896
This commit is contained in:
epriestley
2013-02-17 15:39:36 -08:00
parent 26aac16346
commit bcc082a01e
12 changed files with 460 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
src/applications/diviner/workflow/DivinerAtomizeWorkflow.php
View File

@@ -9,22 +9,29 @@ final class DivinerAtomizeWorkflow extends DivinerWorkflow {
->setArguments(
array(
array(
'name' => 'atomizer',
'param' => 'class',
'help' => 'Specify a subclass of DivinerAtomizer.',
'name' => 'atomizer',
'param' => 'class',
'help' => pht('Specify a subclass of DivinerAtomizer.'),
),
array(
'name' => 'files',
'wildcard' => true,
'name' => 'book',
'param' => 'path',
'help' => pht('Path to a Diviner book configuration.'),
),
array(
'name' => 'ugly',
'help' => 'Produce ugly (but faster) output.',
'name' => 'files',
'wildcard' => true,
),
array(
'name' => 'ugly',
'help' => pht('Produce ugly (but faster) output.'),
),
));
}
public function execute(PhutilArgumentParser $args) {
$this->readBookConfiguration($args);
$console = PhutilConsole::getConsole();
$atomizer_class = $args->getArg('atomizer');
@@ -81,6 +88,11 @@ final class DivinerAtomizeWorkflow extends DivinerWorkflow {
}
$all_atoms = array_mergev($all_atoms);
foreach ($all_atoms as $atom) {
$atom->setBook($this->getConfig('name'));
}
$all_atoms = mpull($all_atoms, 'toDictionary');
$all_atoms = ipull($all_atoms, null, 'hash');

88
src/applications/diviner/workflow/DivinerGenerateWorkflow.php
View File

@@ -2,7 +2,6 @@
final class DivinerGenerateWorkflow extends DivinerWorkflow {
private $config;
private $atomCache;
public function didConstruct() {
@@ -23,10 +22,6 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
));
}
protected function getConfig($key, $default = null) {
return idx($this->config, $key, $default);
}
protected function getAtomCache() {
if (!$this->atomCache) {
$book_root = $this->getConfig('root');
@@ -49,6 +44,7 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
if ($args->getArg('clean')) {
$this->log(pht('CLEARING CACHES'));
$this->getAtomCache()->delete();
$this->log(pht('Done.')."\n");
}
// The major challenge of documentation generation is one of dependency
@@ -134,6 +130,8 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
$this->buildAtomCache();
$this->buildGraphCache();
$this->publishDocumentation();
}
/* -( Atom Cache )--------------------------------------------------------- */
@@ -167,7 +165,7 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
$this->getAtomCache()->saveAtoms();
$this->log(pht("Done."));
$this->log(pht('Done.')."\n");
}
private function getAtomizersForFiles(array $files) {
@@ -252,8 +250,9 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
foreach ($atomizers as $class => $files) {
foreach (array_chunk($files, 32) as $chunk) {
$future = new ExecFuture(
'%s atomize --ugly --atomizer %s -- %Ls',
'%s atomize --ugly --book %s --atomizer %s -- %Ls',
dirname(phutil_get_library_root('phabricator')).'/bin/diviner',
$this->getBookConfigPath(),
$class,
$chunk);
$future->setCWD($this->getConfig('root'));
@@ -352,9 +351,25 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
$this->log(pht('Propagating changes through the graph.'));
foreach ($dirty_symbols as $symbol => $ignored) {
foreach ($atom_cache->getEdgesWithDestination($symbol) as $edge) {
// Find all the nodes which point at a dirty node, and dirty them. Then
// find all the nodes which point at those nodes and dirty them, and so
// on. (This is slightly overkill since we probably don't need to propagate
// dirtiness across documentation "links" between symbols, but we do want
// to propagate it across "extends", and we suffer only a little bit of
// collateral damage by over-dirtying as long as the documentation isn't
// too well-connected.)
$symbol_stack = array_keys($dirty_symbols);
while ($symbol_stack) {
$symbol_hash = array_pop($symbol_stack);
foreach ($atom_cache->getEdgesWithDestination($symbol_hash) as $edge) {
$dirty_nhashes[$edge] = true;
$src_hash = $this->computeSymbolHash($edge);
if (empty($dirty_symbols[$src_hash])) {
$dirty_symbols[$src_hash] = true;
$symbol_stack[] = $src_hash;
}
}
}
@@ -370,7 +385,7 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
$atom_cache->saveEdges();
$atom_cache->saveSymbols();
$this->log(pht('Done.'));
$this->log(pht('Done.')."\n");
}
private function computeSymbolHash($node_hash) {
@@ -386,9 +401,15 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
$atom = $atom_cache->getAtom($node_hash);
$refs = array();
// Make the atom depend on its own symbol, so that all atoms with the same
// symbol are dirtied (e.g., if a codebase defines the function "f()"
// several times, all of them should be dirtied when one is dirtied).
$refs[DivinerAtomRef::newFromDictionary($atom)->toHash()] = true;
foreach (array_merge($atom['extends'], $atom['links']) as $ref_dict) {
$ref = DivinerAtomRef::newFromDictionary($ref_dict);
if ($ref->getProject() == $atom['project']) {
if ($ref->getBook() == $atom['book']) {
$refs[$ref->toHash()] = true;
}
}
@@ -411,42 +432,21 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
return md5(serialize($inputs)).'G';
}
private function readBookConfiguration(PhutilArgumentParser $args) {
$book_path = $args->getArg('book');
if ($book_path === null) {
throw new PhutilArgumentUsageException(
"Specify a Diviner book configuration file with --book.");
}
$book_data = Filesystem::readFile($book_path);
$book = json_decode($book_data, true);
if (!is_array($book)) {
throw new PhutilArgumentUsageException(
"Book configuration '{$book_path}' is not in JSON format.");
}
private function publishDocumentation() {
$atom_cache = $this->getAtomCache();
$graph_map = $atom_cache->getGraphMap();
// If the book specifies a "root", resolve it; otherwise, use the directory
// the book configuration file lives in.
$full_path = dirname(Filesystem::resolvePath($book_path));
if (empty($book['root'])) {
$book['root'] = '.';
}
$book['root'] = Filesystem::resolvePath($book['root'], $full_path);
$this->log(pht('PUBLISHING DOCUMENTATION'));
// Make sure we have a valid book name.
if (!isset($book['name'])) {
throw new PhutilArgumentUsageException(
"Book configuration '{$book_path}' is missing required ".
"property 'name'.");
}
$publisher = new DivinerStaticPublisher();
$publisher->setConfig($this->getAllConfig());
$publisher->setAtomCache($atom_cache);
$publisher->setRenderer(new DivinerDefaultRenderer());
$publisher->publishAtoms(array_values($graph_map));
if (!preg_match('/^[a-z][a-z-]*$/', $book['name'])) {
$name = $book['name'];
throw new PhutilArgumentUsageException(
"Book configuration '{$book_path}' has name '{$name}', but book names ".
"must include only lowercase letters and hyphens.");
}
$this->config = $book;
$this->log(pht('Done.'));
}
}

55
src/applications/diviner/workflow/DivinerWorkflow.php
View File

@@ -2,8 +2,63 @@
abstract class DivinerWorkflow extends PhutilArgumentWorkflow {
private $config;
private $bookConfigPath;
public function getBookConfigPath() {
return $this->bookConfigPath;
}
public function isExecutable() {
return true;
}
protected function getConfig($key, $default = null) {
return idx($this->config, $key, $default);
}
protected function getAllConfig() {
return $this->config;
}
protected function readBookConfiguration(PhutilArgumentParser $args) {
$book_path = $args->getArg('book');
if ($book_path === null) {
throw new PhutilArgumentUsageException(
"Specify a Diviner book configuration file with --book.");
}
$book_data = Filesystem::readFile($book_path);
$book = json_decode($book_data, true);
if (!is_array($book)) {
throw new PhutilArgumentUsageException(
"Book configuration '{$book_path}' is not in JSON format.");
}
// If the book specifies a "root", resolve it; otherwise, use the directory
// the book configuration file lives in.
$full_path = dirname(Filesystem::resolvePath($book_path));
if (empty($book['root'])) {
$book['root'] = '.';
}
$book['root'] = Filesystem::resolvePath($book['root'], $full_path);
// Make sure we have a valid book name.
if (!isset($book['name'])) {
throw new PhutilArgumentUsageException(
"Book configuration '{$book_path}' is missing required ".
"property 'name'.");
}
if (!preg_match('/^[a-z][a-z-]*$/', $book['name'])) {
$name = $book['name'];
throw new PhutilArgumentUsageException(
"Book configuration '{$book_path}' has name '{$name}', but book names ".
"must include only lowercase letters and hyphens.");
}
$this->bookConfigPath = $book_path;
$this->config = $book;
}
}