/*
TODO:
* show attributes on declaration (including privacy, @nogc, and UDAs, extern(C), @property, etc.)
* add type class tag to the navbar
* better format of contracts
* Invariants for structs etc
* old-style alias implementation
* inheritance checking
* more cross-linking in examples and prototypes
* package listing
* the ``` syntax
- BLOCKQUOTE too
- maybe markdown list style
* the // just docs thing
* table of contents in longer documents for sections
* automatic scan and generation of a whole directory
* write all the other linked documents
* make the search better
* bugs
- Throws:
- make REF smarter
- make the highlighter run inside certain macros
* clean up this hideously awful code
- make it easier for a user thing to replace the skeleton
* whatever other cool little things we want
- link to source exactly where it is found
- versioning
* Template constraint analyzer
* The suggestion box and preview uploader. Third party libs can contribute too.
BUG:
the PSYMBOL _ thing is wrong in a lot of cases (see std.format's wiki link)
*/
/*
This program is released under the GPL and heavily uses
components by Brian Schott (aka Hackerpilot), including
some of his GPL code.
// FIXME: maybe the details page should show an example import statement to get to the item
// FIXME: any ditto comments should automatically tie the functions together under see also
Magic syntax (NOT YET IMPLEMENTED):
`inline code`
Inline code does not get further magic processing.
It is processed on the first step. Empty inline code
is kept unmodified, including the `. It cannot span
across a line.
``referenced symbol``
If possible, it is looked up in scope, otherwise
it is assumed to be a sub name of the current item.
*/
/*
Using DCD code, I can resolve the symbols. I want to ask it to fetch the docs
for the symbol under the cursor which it should do with the autocomplete list.
If I scan for all throw statements in the AST, I can figure out what a function throws.
*/
/// my doc thing
module doc;
import dparse.parser;
import dparse.lexer;
import dparse.formatter;
import std.algorithm : canFind, sort;
enum bool skip_private = true;
enum bool skip_undocumented = true;
const(char)[] htmlEncode(const(char)[] s) {
return s.
replace("&", "&").
replace("<", "<").
replace(">", ">");
}
void writeln(T...)(T t) {
import std.stdio : stdout;
stdout.writeln(t);
}
// From Dscanner
import symbol_finder;
import std.file;
/*
Params:
The line, excluding whitespace, must start with
identifier = to start a new thing. If a new thing starts,
it closes the old.
The description may be formatted however. Whitespace is stripped.
*/
string getIdent(T)(T t) {
if(t is null)
return null;
if(t.identifier == tok!"")
return null;
return t.identifier.text;
}
bool hasParam(T)(const T dec, string name) {
if(dec is null)
return false;
if(dec.parameters && dec.parameters.parameters)
foreach(parameter; dec.parameters.parameters)
if(parameter.name.text == name)
return true;
if(dec.templateParameters && dec.templateParameters.templateParameterList)
foreach(parameter; dec.templateParameters.templateParameterList.items) {
if(getIdent(parameter.templateTypeParameter) == name)
return true;
if(getIdent(parameter.templateValueParameter) == name)
return true;
if(getIdent(parameter.templateAliasParameter) == name)
return true;
if(getIdent(parameter.templateTupleParameter) == name)
return true;
if(parameter.templateThisParameter && getIdent(parameter.templateThisParameter.templateTypeParameter) == name)
return true;
}
return false;
}
struct DocComment {
string synopsis;
string details;
string[] params; // stored as ident=txt. You can split on first index of =.
string[] throws;
string returns;
string examples;
string diagnostics;
string[] see_alsos;
string[string] otherSections;
string[] fullyQualifiedName;
void writeSynopsis(Documenter.MyOutputRange output) {
output.putTag("
");
}
import std.typecons : Tuple;
void writeDetails(T = FunctionDeclaration)(Documenter.MyOutputRange output) {
writeDetails!T(output, cast(T) null, null);
}
void writeDetails(T = FunctionDeclaration)(Documenter.MyOutputRange output, const T functionDec = null, Tuple!(string, string)[] utInfo = null) {
auto f = new MyFormatter!(typeof(output))(output);
if(params.length) {
output.putTag("Parameters
");
output.putTag("");
foreach(param; params) {
auto split = param.indexOf("=");
auto paramName = param[0 .. split];
if(!hasParam(functionDec, paramName))
continue;
output.putTag("- ");
output.putTag("");
output.put(param[0 .. split]);
output.putTag("");
output.putTag("
");
output.putTag("- ");
if(functionDec !is null) {
const(Parameter)* paramAst;
foreach(ref p; functionDec.parameters.parameters) {
if(p.name.type != tok!"")
if(p.name.text == param[0 .. split]) {
paramAst = &p;
break;
}
}
if(paramAst) {
output.putTag("
");
output.putTag("Type: ");
output.putTag("");
f.format(paramAst.type);
output.putTag("");
output.putTag("
");
}
}
output.putTag(formatDocumentationComment(param[split + 1 .. $], fullyQualifiedName));
output.putTag(" ");
}
output.putTag("
");
}
static if(!is(T == Constructor))
if(returns !is null) {
output.putTag("Return Value
");
output.putTag("");
if(functionDec !is null) {
output.putTag("
");
output.putTag("Type: ");
output.putTag("
");
if(functionDec.hasAuto && functionDec.hasRef)
output.putTag(`auto ref `);
else {
if (functionDec.hasAuto)
output.putTag(`auto `);
if (functionDec.hasRef)
output.putTag(`ref `);
}
if (functionDec.returnType !is null)
f.format(functionDec.returnType);
output.putTag("");
output.putTag("
");
}
output.putTag(formatDocumentationComment(returns, fullyQualifiedName));
output.putTag("
");
}
if(details.strip.length) {
output.putTag("Detailed Description
");
output.putTag("");
}
if(diagnostics.strip.length) {
output.putTag("Diagnostics
");
output.putTag("");
}
if(examples.length || utInfo.length) {
output.putTag("Examples
");
output.putTag(formatDocumentationComment(examples, fullyQualifiedName));
foreach(example; utInfo) {
output.putTag("");
output.putTag(formatDocumentationComment(preprocessComment(example[1]), fullyQualifiedName));
output.putTag("
");
output.putTag("");
output.putTag(highlight(outdent(example[0])));
output.putTag("
");
}
}
if(see_alsos.length) {
output.putTag("See Also
");
output.putTag("");
foreach(line; see_alsos) {
output.putTag("- ");
output.putTag(formatDocumentationComment(line, fullyQualifiedName));
output.putTag("
");
}
output.putTag("
");
}
if(otherSections.keys.length) {
output.putTag("");
}
foreach(section, content; otherSections) {
output.putTag("");
output.putTag("
");
output.put(section.capitalize);
output.putTag("
");
output.putTag(formatDocumentationComment(content, fullyQualifiedName));
output.putTag("");
}
}
}
string preprocessComment(string comment) {
if(comment.length < 3)
return comment;
comment = comment[1 .. $]; // trim off the /
auto commentType = comment[0];
while(comment.length && comment[0] == commentType)
comment = comment[1 .. $]; // trim off other opening spam
string closingSpam;
if(commentType == '*' || commentType == '+') {
comment = comment[0 .. $-1]; // trim off the closing /
bool closingSpamFound;
while(comment.length && comment[$-1] == commentType) {
comment = comment[0 .. $-1]; // trim off other closing spam
closingSpamFound = true;
}
if(closingSpamFound) {
// if there was closing spam we also want to count the spaces before it
// to trim off other line spam. The goal here is to clean up
/**
* Resolve host name.
* Returns: false if unable to resolve.
*/
// into just "Resolve host name.\n Returns: false if unable to resolve."
// give or take some irrelevant whitespace.
while(comment.length && (comment[$-1] == '\t' || comment[$-1] == ' ')) {
closingSpam = comment[$-1] ~ closingSpam;
comment = comment[0 .. $-1]; // trim off other closing spam
}
if(closingSpam.length == 0)
closingSpam = " "; // some things use the " *" leader, but still end with "*/" on a line of its own
}
}
string poop;
if(commentType == '/')
poop = "/// ";
else
poop = closingSpam ~ commentType ~ " ";
string newComment;
foreach(line; comment.splitter("\n")) {
// check for " * some text"
if(line.length >= poop.length && line.startsWith(poop))
newComment ~= line[poop.length .. $];
// check for an empty line with just " *"
else if(line.length == poop.length-1 && line[0..poop.length-1] == poop[0..$-1])
{} // this space is intentionally left blank; it is an empty line
else
newComment ~= line;
newComment ~= "\n";
}
comment = newComment;
return comment;
}
DocComment parseDocumentationComment(string comment, string[] fullyQualifiedName) {
DocComment c;
c.fullyQualifiedName = fullyQualifiedName;
comment = preprocessComment(comment);
void parseSections(string comment) {
string remaining;
string section;
bool inSynopsis = true;
bool justSawBlank = false;
bool inCode = false;
bool hasAnySynopsis = false;
foreach(line; comment.splitter("\n")) {
auto maybe = line.strip.toLower;
if(maybe.startsWith("---")) {
justSawBlank = false;
inCode = !inCode;
}
if(inCode){
justSawBlank = false;
goto ss; // sections never change while in a code example
}
if(inSynopsis && hasAnySynopsis && maybe.length == 0) {
// two blank lines in a row ends the synopsis
if(justSawBlank)
inSynopsis = false;
justSawBlank = true;
} else {
justSawBlank = false;
}
if(maybe.startsWith("params:")) {
section = "params";
inSynopsis = false;
} else if(maybe.startsWith("returns:")) {
section = "returns";
line = line[line.indexOf(":")+1 .. $];
inSynopsis = false;
} else if(maybe.startsWith("throws:")) {
section = "throws";
inSynopsis = false;
} else if(maybe.startsWith("author:")) {
section = "authors";
line = line[line.indexOf(":")+1 .. $];
inSynopsis = false;
} else if(maybe.startsWith("authors:")) {
section = "authors";
line = line[line.indexOf(":")+1 .. $];
inSynopsis = false;
} else if(maybe.startsWith("source:")) {
section = "source";
line = line[line.indexOf(":")+1 .. $];
inSynopsis = false;
} else if(maybe.startsWith("version:")) {
section = "version";
line = line[line.indexOf(":")+1 .. $];
inSynopsis = false;
} else if(maybe.startsWith("license:")) {
section = "license";
line = line[line.indexOf(":")+1 .. $];
inSynopsis = false;
} else if(maybe.startsWith("copyright:")) {
section = "copyright";
line = line[line.indexOf(":")+1 .. $];
inSynopsis = false;
} else if(maybe.startsWith("see_also:")) {
section = "see_also";
inSynopsis = false;
line = line[line.indexOf(":")+1 .. $];
} else if(maybe.startsWith("diagnostics:")) {
inSynopsis = false;
section = "diagnostics";
line = line[line.indexOf(":")+1 .. $];
} else if(maybe.startsWith("examples:")) {
inSynopsis = false;
section = "examples";
line = line[line.indexOf(":")+1 .. $];
} else if(maybe.startsWith("example:")) {
inSynopsis = false;
line = line[line.indexOf(":")+1 .. $];
section = "examples"; // Phobos uses example, the standard is examples.
} else if(maybe.startsWith("version:")) {
inSynopsis = false;
line = line[line.indexOf(":")+1 .. $];
section = "version";
} else if(maybe.startsWith("standards:")) {
inSynopsis = false;
section = "standards";
} else if(maybe.startsWith("history:")) {
inSynopsis = false;
section = "history";
} else if(maybe.startsWith("deprecated:")) {
inSynopsis = false;
section = "deprecated";
} else if(maybe.startsWith("date:")) {
inSynopsis = false;
section = "date";
} else if(maybe.startsWith("bugs:")) {
inSynopsis = false;
section = "bugs";
} else if(maybe.startsWith("macros:")) {
inSynopsis = false;
section = "macros";
} else {
// no change to section
}
ss: switch(section) {
case "params":
bool lookingForIdent = true;
bool inIdent;
bool skippingSpace;
size_t space_at;
auto lol = line.strip;
foreach(idx, ch; lol) {
import std.uni, std.ascii : isAlphaNum;
if(lookingForIdent && !inIdent) {
if(!isAlpha(ch) && ch != '_')
continue;
inIdent = true;
lookingForIdent = false;
}
if(inIdent) {
if(ch == '_' || isAlphaNum(ch) || isAlpha(ch))
continue;
else {
skippingSpace = true;
inIdent = false;
space_at = idx;
}
}
if(skippingSpace) {
if(!isWhite(ch)) {
if(ch == '=') {
// we finally hit a thingy
c.params ~= lol[0 .. space_at] ~ "=" ~ lol[idx + 1 .. $] ~ "\n";
break ss;
} else
// we are expecting whitespace or = and hit
// neither.. this can't be ident = desc!
break;
}
}
}
if(c.params.length)
c.params[$-1] ~= line ~ "\n";
break;
case "see_also":
auto s = line.strip;
if(s.length)
c.see_alsos ~= s;
break;
case "macros":
// ignoring for now
break;
case "returns":
c.returns ~= line ~ "\n";
break;
case "diagnostics":
c.diagnostics ~= line ~ "\n";
break;
case "authors":
case "license":
case "source":
case "copyright":
case "version":
c.otherSections[section] ~= line ~ "\n";
break;
case "examples":
c.examples ~= line ~ "\n";
break;
default:
if(inSynopsis) {
c.synopsis ~= line ~ "\n";
if(line.length)
hasAnySynopsis = true;
} else
remaining ~= line ~ "\n";
}
}
c.synopsis = formatDocumentationComment(c.synopsis, fullyQualifiedName);
c.details = formatDocumentationComment(remaining, fullyQualifiedName);
}
parseSections(comment);
return c;
}
string formatDocumentationComment(string comment, string[] fullyQualifiedName) {
// remove that annoying ddocism to suppress its auto-highlight anti-feature
auto psymbols = fullyQualifiedName[$-1].split(".");
// also trim off our .# for overload
import std.uni :isAlpha;
auto psymbol = (psymbols.length && psymbols[$-1].length) ?
((psymbols[$-1][0].isAlpha || psymbols[$-1][0] == '_') ? psymbols[$-1] : psymbols[$-2])
:
null;
if(psymbol.length)
comment = comment.replace("_" ~ psymbol, psymbol);
OutBuffer buf;
buf.data = comment.dup;
highlightParagraphs(null, &buf, 0, buf.offset);
auto data = cast(string) buf.data;
// FIXME: wrong
while(1) {
auto idx = data.indexOf("");
if(idx == -1)
break;
auto idxCode = idx + "".length;
auto idxEnd = data[idx .. $].indexOf("
");
if(idxEnd == -1)
assert(0, data[]);//idx .. $]);
idxEnd += idx;
data = data[0 .. idx] ~
"" ~
highlight(outdent(data[idxCode .. idxEnd])) ~ data[idxEnd .. $];
}
string fqn;
foreach(idx, name; fullyQualifiedName) {
if(idx) fqn ~= ".";
fqn ~= name;
}
data = data.expandDdocMacros([
"FULLY_QUALIFIED_NAME" : fqn,
"MODULE_NAME" : fullyQualifiedName[0],
"AMP" : "&",
"LT" : "<",
"GT" : ">",
"DOLLAR" : "$",
"BACKTICK" : "`",
"COMMA": ",",
"ARGS" : "$0",
"H1" : "$0
",
"H2" : "$0
",
"H3" : "$0
",
"H4" : "$0
",
"H5" : "$0
",
"H6" : "$0
",
// support for my docs
"M" : "$(LREF $0)",
"L" : "$0",
"DDOC_ANCHOR" : "$0",
"TIP" : "$0
",
"NOTE" : "$0
",
"WARNING" : "$0
",
"PITFALL" : "$0
",
"SIDEBAR" : "",
// this is support for the Phobos docs
"D" : "$0", // this is magical! D is actually handled in code.
"REF" : `$0`, // this is magical! Handles ref in code
"D_PARAM" : "$(D $0)",
"I" : "$0",
"B" : "$0",
"P" : "$0
",
"LINK2" : "$2",
"SUBMODULE" : `$0`,
"SUBREF" : `$2`,
"SHORTXREF" : `$2`,
"SHORTXREF_PACK" : `$3`,
"XREF" : `std.$1.$2`,
"CXREF" : `core.$1.$2`,
"XREF_PACK" : `std.$1.$2.$3`,
"MYREF" : `$0`,
"LREF" : `$0`,
"MREF" : `$(REF $1,$+)`,
"BIGOH" : "O($0)",
"TR" : "$0
",
"TH" : "$0 | ",
"TD" : "$0 | ",
"TDNW" : "$0 | ",
"UL" : "
",
"OL" : "$0
",
"LI" : "$0",
"SCRIPT" : "",
"DIVC" : `$+
`,
"BOOKTABLE" : "",
"T2" : "$(LREF $1) | $+ |
",
"WEB" : `$2`,
"XREF_PACK_NAMED" : `$4`,
// std.regex
"REG_ROW":`$(TR $(TD $(I $1 )) $(TD $+) )`,
"REG_TITLE":`$(TR $(TD $(B $1)) $(TD $(B $2)) )`,
"REG_TABLE":``,
"REG_START":` $0
`,
"SECTION":``,
"S_LINK":`$+`,
],
[ // number of arguments expected, if needed
"BOOKTABLE" : 1,
"T2" : 1,
"WEB" : 2,
"XREF_PACK_NAMED" : 4,
"DIVC" : 1,
"SUBREF" : 2,
"LINK2" : 2,
"XREF" : 2,
"CXREF" : 2,
"SHORTXREF" : 2,
"SHORTXREF_PACK" : 3,
"XREF_PACK" : 3,
"MREF" : 1,
"REG_ROW" : 1,
"REG_TITLE" : 2,
"S_LINK" : 1,
]);
return data;
}
enum outputDir = "/var/www/dpldocs.info/experimental-docs/";
struct AdditionalModuleInfo {
import unittest_preprocessor;
TestRange[][size_t] unittestMapping;
const(ASTNode)[][const(ASTNode)] dittos;
const(ASTNode)[][const(ASTNode)] overloadsOf;
const(ubyte)[] fileBytes;
string[][string] childrenList;
}
string[const(ASTNode)] linkMapping;
string[const(ASTNode)] nameMapping;
void main(string[] args) {
LexerConfig config;
StringCache stringCache = StringCache(128);
config.stringBehavior = StringBehavior.source;
config.whitespaceBehavior = WhitespaceBehavior.include;
Module[] modules;
AdditionalModuleInfo[] additionalModuleInfo;
import dsymbol.conversion : ScopeSymbolPair, ModuleCache;
import std.experimental.allocator : allocatorObject;
import std.experimental.allocator.mallocator : Mallocator;
ScopeSymbolPair[] symbols;
ModuleCache cache = ModuleCache(allocatorObject(Mallocator.instance));
//cache.addImportPaths(["/home/me/d/pull-request-stuff/druntime/import"]);
cache.addImportPaths(["."], false);
// first pass figures out dittos and overloads
foreach(argIdx, arg; args[1 .. $]) {
try {
writeln("First pass processing ", arg);
auto b = cast(ubyte[]) read(arg);
config.fileName = arg;
auto tokens = getTokensForParser(b, config, &stringCache);
if(0){
cache.cacheModule(arg);
import dsymbol.conversion;
import dsymbol.string_interning;
symbols ~= generateAutocompleteTrees(tokens, allocatorObject(Mallocator.instance), 0, cache);
auto bullshit = symbols[$-1].scope_;
auto stuff = bullshit.getSymbolsByName(internString("foo"))[0];
writeln(stuff.symbolFile ~ " has " ~stuff.name);
if(argIdx)
return;
}
import std.path : baseName;
auto m = parseModule(tokens, baseName(arg));
modules ~= m;
auto outliner = new Looker(b, baseName(arg));
outliner.visit(m);
additionalModuleInfo ~= AdditionalModuleInfo(
outliner.unittestMapping,
outliner.dittos,
outliner.overloadsOf,
outliner.fileBytes,
outliner.childrenList
);
} catch(Throwable t) {
modules ~= null;
writeln(t);
}
}
// second pass does the actual work
foreach(idx, arg; args[1 .. $]) {
try {
if(modules[idx] is null)
continue;
writeln("Second pass processing ", arg);
import std.path : baseName;
auto outliner = new Documenter(outputDir, additionalModuleInfo[idx], baseName(arg));
outliner.visit(modules[idx]);
.destroy(outliner);
} catch(Throwable t) { writeln(t); }
}
}
class MyFormatter(Sink) : Formatter!Sink {
this(Sink sink, bool useTabs = true, IndentStyle style = IndentStyle.otbs, uint indentWidth = 4)
{
super(sink, useTabs, style, indentWidth);
}
void putTag(in char[] s) {
static if(!__traits(compiles, sink.putTag("")))
sink.put(s);
else
sink.putTag(s);
}
override void put(string s1) {
auto s = cast(const(char)[]) s1;
static if(!__traits(compiles, sink.putTag("")))
sink.put(s.htmlEncode);
else
sink.put(s);
}
override void format(const TemplateParameterList templateParameterList)
{
foreach(i, param; templateParameterList.items)
{
putTag("");
put("\t");
format(param);
putTag("
");
}
}
override void format(const InStatement inStatement) {
putTag("");
put("in");
putTag("");
//put(" ");
format(inStatement.blockStatement);
}
override void format(const OutStatement outStatement) {
putTag("");
put("out");
putTag("");
if (outStatement.parameter != tok!"")
{
put(" (");
format(outStatement.parameter);
put(")");
}
//put(" ");
format(outStatement.blockStatement);
}
override void format(const AssertExpression assertExpression)
{
debug(verbose) writeln("AssertExpression");
/**
AssignExpression assertion;
AssignExpression message;
**/
with(assertExpression)
{
putTag("");
put("assert");
putTag(" (");
format(assertion);
if (message)
{
put(", ");
format(message);
}
put(")");
}
}
override void format(const TemplateAliasParameter templateAliasParameter)
{
debug(verbose) writeln("TemplateAliasParameter");
/**
Type type;
Token identifier;
Type colonType;
AssignExpression colonExpression;
Type assignType;
AssignExpression assignExpression;
**/
with(templateAliasParameter)
{
putTag("");
put("alias");
putTag("");
put(" ");
if (type)
{
format(type);
space();
}
format(identifier);
if (colonType)
{
put(" : ");
format(colonType);
}
else if (colonExpression)
{
put(" : ");
format(colonExpression);
}
if (assignType)
{
put(" = ");
format(assignType);
}
else if (assignExpression)
{
put(" = ");
format(assignExpression);
}
}
}
override void format(const Constraint constraint)
{
debug(verbose) writeln("Constraint");
if (constraint.expression)
{
put(" ");
putTag("");
put("if");
putTag("");
put(" (");
putTag("");
format(constraint.expression);
putTag("
");
put(")");
}
}
override void format(const PrimaryExpression primaryExpression)
{
debug(verbose) writeln("PrimaryExpression");
/**
Token dot;
Token primary;
IdentifierOrTemplateInstance identifierOrTemplateInstance;
Token basicType;
TypeofExpression typeofExpression;
TypeidExpression typeidExpression;
ArrayLiteral arrayLiteral;
AssocArrayLiteral assocArrayLiteral;
Expression expression;
IsExpression isExpression;
LambdaExpression lambdaExpression;
FunctionLiteralExpression functionLiteralExpression;
TraitsExpression traitsExpression;
MixinExpression mixinExpression;
ImportExpression importExpression;
Vector vector;
**/
with(primaryExpression)
{
if (dot != tok!"") put(".");
if (basicType != tok!"") format(basicType);
if (primary != tok!"")
{
//put("SWEET/");
if (basicType != tok!"") put("."); // i.e. : uint.max
format(primary);
//put("/SWEET");
}
if (expression)
{
put("(");
format(expression);
put(")");
}
else if (identifierOrTemplateInstance)
{
format(identifierOrTemplateInstance);
}
else if (typeofExpression) format(typeofExpression);
else if (typeidExpression) format(typeidExpression);
else if (arrayLiteral) format(arrayLiteral);
else if (assocArrayLiteral) format(assocArrayLiteral);
else if (isExpression) format(isExpression);
else if (lambdaExpression) format(lambdaExpression);
else if (functionLiteralExpression) format(functionLiteralExpression);
else if (traitsExpression) format(traitsExpression);
else if (mixinExpression) format(mixinExpression);
else if (importExpression) format(importExpression);
else if (vector) format(vector);
}
}
override void format(const IsExpression isExpression) {
//Formatter!Sink.format(this);
with(isExpression) {
putTag(`is(`);
if (type) format(type);
if (identifier != tok!"") {
space();
format(identifier);
}
if (equalsOrColon) {
space();
put(tokenRep(equalsOrColon));
space();
}
if (typeSpecialization) format(typeSpecialization);
if (templateParameterList) {
put(", ");
format(templateParameterList);
}
put(")");
}
}
override void format(const TypeofExpression typeofExpr) {
debug(verbose) writeln("TypeofExpression");
/**
Expression expression;
Token return_;
**/
putTag("typeof(");
typeofExpr.expression ? format(typeofExpr.expression) : format(typeofExpr.return_);
put(")");
}
override void format(const Parameter parameter)
{
debug(verbose) writeln("Parameter");
/**
IdType[] parameterAttributes;
Type type;
Token name;
bool vararg;
AssignExpression default_;
TypeSuffix[] cstyle;
**/
putTag("");
foreach (count, attribute; parameter.parameterAttributes)
{
if (count) space();
putTag("
");
put(tokenRep(attribute));
putTag("");
}
if (parameter.parameterAttributes.length > 0)
space();
if (parameter.type !is null)
format(parameter.type);
if (parameter.name.type != tok!"")
{
space();
putTag(`
`);
put(parameter.name.text);
putTag("");
}
foreach(suffix; parameter.cstyle)
format(suffix);
if (parameter.default_)
{
put(" = ");
format(parameter.default_);
}
if (parameter.vararg) {
putTag("
");
put("...");
putTag("");
}
putTag("
");
}
override void format(const Type type)
{
debug(verbose) writeln("Type(");
/**
IdType[] typeConstructors;
TypeSuffix[] typeSuffixes;
Type2 type2;
**/
foreach (count, constructor; type.typeConstructors)
{
if (count) space();
put(tokenRep(constructor));
}
if (type.typeConstructors.length) space();
//put(``);
format(type.type2);
//put("");
foreach (suffix; type.typeSuffixes)
format(suffix);
debug(verbose) writeln(")");
}
override void format(const Type2 type2)
{
debug(verbose) writeln("Type2");
/**
IdType builtinType;
Symbol symbol;
TypeofExpression typeofExpression;
IdentifierOrTemplateChain identifierOrTemplateChain;
IdType typeConstructor;
Type type;
**/
if (type2.symbol !is null)
{
putTag("");
format(type2.symbol);
putTag("");
}
else if (type2.typeofExpression !is null)
{
format(type2.typeofExpression);
if (type2.identifierOrTemplateChain)
{
put(".");
format(type2.identifierOrTemplateChain);
}
return;
}
else if (type2.typeConstructor != tok!"")
{
putTag("");
put(tokenRep(type2.typeConstructor));
putTag("");
put("(");
format(type2.type);
put(")");
}
else
{
putTag("");
put(tokenRep(type2.builtinType));
putTag("");
}
}
override void format(const StorageClass storageClass)
{
debug(verbose) writeln("StorageClass");
/**
AtAttribute atAttribute;
Deprecated deprecated_;
LinkageAttribute linkageAttribute;
Token token;
**/
with(storageClass)
{
if (atAttribute) format(atAttribute);
else if (deprecated_) format(deprecated_);
else if (linkageAttribute) format(linkageAttribute);
else {
putTag("");
format(token);
putTag("");
}
}
}
bool noTag;
override void format(const Token token)
{
debug(verbose) writeln("Token ", tokenRep(token));
if(!noTag && token == tok!"identifier") {
putTag("");
putTag(``);
}
put(tokenRep(token));
if(!noTag && token == tok!"identifier") {
putTag("");
putTag("");
}
}
version(none)
override void format(const IdentifierOrTemplateInstance identifierOrTemplateInstance)
{
debug(verbose) writeln("IdentifierOrTemplateInstance");
putTag("");
with(identifierOrTemplateInstance)
{
format(identifier);
if (templateInstance)
format(templateInstance);
}
putTag("");
}
version(none)
override void format(const Symbol symbol)
{
debug(verbose) writeln("Symbol");
put("GOOD/");
if (symbol.dot)
put(".");
format(symbol.identifierOrTemplateChain);
put("/GOOD");
}
override void format(const Parameters parameters)
{
debug(verbose) writeln("Parameters");
/**
Parameter[] parameters;
bool hasVarargs;
**/
put("(");
putTag("");
foreach (count, param; parameters.parameters)
{
if (count) put("\n");
put("\t");
format(param);
}
if (parameters.hasVarargs)
{
if (parameters.parameters.length)
put("\n");
putTag("
");
}
putTag("
");
put(")");
}
override void format(const IdentifierChain identifierChain)
{
//put("IDENT");
foreach(count, ident; identifierChain.identifiers)
{
if (count) put(".");
put(ident.text);
}
//put("/IDENT");
}
override void format(const AndAndExpression andAndExpression)
{
with(andAndExpression)
{
putTag("");
format(left);
if (right)
{
putTag(" &&
");
format(right);
}
putTag("
");
}
}
override void format(const OrOrExpression orOrExpression)
{
with(orOrExpression)
{
putTag("");
format(left);
if (right)
{
putTag(" ||
");
format(right);
}
putTag("
");
}
}
alias format = Formatter!Sink.format;
}
// The following code is based on stuff by Hackerpilot
// Original Copyright Brian Schott (Hackerpilot) 2014.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
import dparse.lexer;
import dparse.ast;
import dparse.formatter;
static import std.stdio;
import std.string;
import std.array;
import std.conv;
/*
This needs to be redone. It needs to have a first pass that collects the module info
and a second pass that uses it without actually visiting the AST again.
First pass:
run through all files and collect their module names. Make a map
of module name to AST.
Second pass:
run through all files and generate their scope information
To process a scope:
1) Add local names to the list
2) do the imports and add imported names to the list.
3) run through inheritance tress and add those names
struct Symbol {
ASTNode this_;
Symbol[] children;
enum How {
native,
inherited,
imported
}
How how;
}
Third pass:
actually generate the documentation from the preprocessed symbols.
*/
// We need to find dittos, overloads, and unittest maps in a separate pass
// so we can go ahead and link them in later.
//
// It should prolly also create the nav maps now too
class Looker : ASTVisitor {
alias visit = ASTVisitor.visit;
string[] lastCommentChain;
int[string][] overloadChain; // FIXME: the doc needs to explain overload sets too
ASTNode[] dittoChain; // no Object ref ugh so casting
ASTNode[] overloadNodeChain;
string[] fullyQualifiedName;
// I need to keep the unittests, the dittos, and the overloads.
// we also want a map.
TestRange[][size_t] unittestMapping;
const(ASTNode)[][const(ASTNode)] dittos;
const(ASTNode)[][const(ASTNode)] overloadsOf;
string[][string] childrenList;
string getFileName() {
string fn = "";
foreach(idx, name; fullyQualifiedName) {
if(idx) fn ~= ".";
fn ~= name;
}
fn ~= ".html";
return fn;
}
string getFqn(string f = null) {
string fn = "";
foreach(idx, name; fullyQualifiedName) {
if(idx) fn ~= ".";
fn ~= name;
}
if(f.length && fn.length)
fn ~= ".";
fn ~= f;
return fn;
}
string dittoSupport(const ASTNode node, string comment) {
linkMapping[node] = getFileName();
nameMapping[node] = getFqn();
auto c = preprocessComment(comment).toLower.strip;
if(lastCommentChain.length == 0) {
lastCommentChain ~= null;
dittoChain ~= null;
}
if(c == "ditto") {
/*
/// foo
version(a) {
int a; /// ditto
}
that will trigger the below
*/
if(dittoChain[$-1] is null)
return comment; // FIXME
//assert(0);
if(dittoChain[$-1] !in dittos)
dittos[dittoChain[$-1]] ~= dittoChain[$-1];
dittos[dittoChain[$-1]] ~= node;
return lastCommentChain[$-1];
} else if(c.length) {
dittoChain[$-1] = cast() node;
return (lastCommentChain[$-1] = comment);
}
else return comment;
}
void descendInto(string name) {
assert(name.length);
if(getFqn !in childrenList || !childrenList[getFqn].canFind(getFqn(name)))
childrenList[getFqn] ~= getFqn(name);
fullyQualifiedName ~= name;
lastCommentChain ~= null;
dittoChain ~= null;
overloadChain ~= null;
overloadNodeChain ~= null;
}
void ascendOutOf(string name) {
fullyQualifiedName = fullyQualifiedName[0 .. $-1];
if(lastCommentChain.length)
lastCommentChain = lastCommentChain[0 .. $-1];
if(overloadChain.length) {
overloadChain = overloadChain[0 .. $-1];
overloadNodeChain = overloadNodeChain[0 .. $-1];
}
if(dittoChain.length)
dittoChain = dittoChain[0 .. $-1];
}
const(ubyte)[] fileBytes;
string originalFileName;
this(const(ubyte)[] fileBytes, string fileName)
{
this.fileBytes = fileBytes;
this.originalFileName = fileName;
}
import unittest_preprocessor;
override void visit(const Module mod) {
unittestMapping = getUnittestMap(mod);
//output.putTag("");
string moduleName;
DocComment comment;
if(mod.moduleDeclaration !is null) {
auto moduleDec = mod.moduleDeclaration;
auto app = appender!(char[])();
auto f = new MyFormatter!(typeof(app))(app);
f.format(moduleDec.moduleName);
moduleName = app.data.idup;
}
assert(moduleName.length); // FIXME????
descendInto(moduleName);
mod.accept(this);
ascendOutOf(fullyQualifiedName[0]);
}
override void visit(const FunctionDeclaration functionDec) {
doFunctionDec(functionDec);
}
override void visit(const Constructor c) {
doFunctionDec(c);
}
// FIXME: make sure template declaration handles overload chain too
void doFunctionDec(T)(const T functionDec) {
static if(is(T == Constructor))
string name = "this";
else
string name = functionDec.name.text;
if(functionDec.comment.length == 0 && skip_undocumented)
return;
if(auto ptr = name in overloadChain[$-1]) {
*ptr += 1;
name ~= "." ~ to!string(*ptr);
if(overloadNodeChain[$-1] !in overloadsOf)
overloadsOf[overloadNodeChain[$-1]] ~= overloadNodeChain[$-1];
overloadsOf[overloadNodeChain[$-1]] ~= functionDec;
} else {
overloadChain[$-1][name] = 1;
overloadNodeChain[$-1] = cast() functionDec;
}
//fullyQualifiedName ~= name;
descendInto(name);
auto comment = parseDocumentationComment(dittoSupport(functionDec, functionDec.comment), fullyQualifiedName ~ name);
ascendOutOf(name);
//fullyQualifiedName = fullyQualifiedName[0..$-1];
}
override void visit(const StructDeclaration structDec) {
doAggregate(structDec);
}
override void visit(const ClassDeclaration classDec) {
doAggregate(classDec);
}
override void visit(const UnionDeclaration unionDec) {
doAggregate(unionDec);
}
override void visit(const InterfaceDeclaration interfaceDec) {
doAggregate(interfaceDec);
}
override void visit(const TemplateDeclaration templateDec) {
doAggregate(templateDec);
}
// FIXME: alias declaration
// FIXME: template mixin expression
void doAggregate(T)(const T classDec)
{
static if(is(T == ClassDeclaration))
auto aggregateType = "class";
else static if(is(T == InterfaceDeclaration))
auto aggregateType = "interface";
else static if(is(T == StructDeclaration))
auto aggregateType = "struct";
else static if(is(T == TemplateDeclaration))
auto aggregateType = "template";
else static if(is(T == UnionDeclaration))
auto aggregateType = "union";
else static assert(0);
if(classDec.comment.length == 0 && skip_undocumented)
return;
auto name = classDec.name.text;
if(name.length == 0)
return; // skipping anonymous structs and unions.. prolly worthy of a FIXME
fullyQualifiedName ~= name;
auto comment = parseDocumentationComment(dittoSupport(classDec, classDec.comment), fullyQualifiedName);
fullyQualifiedName = fullyQualifiedName[0 .. $-1];
descendInto(name);
classDec.accept(this);
ascendOutOf(name);
}
override void visit(const EnumDeclaration enumDec) {
doEnumDecl(enumDec);
}
override void visit(const AnonymousEnumDeclaration enumDec) {
doEnumDecl(enumDec);
}
void doEnumDecl(T)(const T enumDec) {
static if(is(T == AnonymousEnumDeclaration)) {
if(enumDec.members.length == 0) return;
auto name = enumDec.members[0].name.text;
auto type = enumDec.baseType;
auto members = enumDec.members;
} else {
auto name = enumDec.name.text;
auto type = enumDec.type;
const(EnumMember)[] members;
if(enumDec.enumBody)
members = enumDec.enumBody.enumMembers;
}
static if(is(T == AnonymousEnumDeclaration)) {
// undocumented anonymous enums get a pass if any of
// their members are documented because that's what dmd does...
// FIXME maybe
bool foundOne = false;
foreach(member; members) {
if(member.comment.length) {
foundOne = true;
break;
}
}
if(!foundOne && skip_undocumented)
return;
} else {
if(enumDec.comment.length == 0 && skip_undocumented)
return;
}
descendInto(name);
auto comment = parseDocumentationComment(dittoSupport(enumDec, enumDec.comment), fullyQualifiedName ~ name);
foreach(member; members) {
//output.put(member.name.text);
}
ascendOutOf(name);
}
override void visit(const VariableDeclaration variableDeclaration)
{
foreach (const Declarator d; variableDeclaration.declarators)
{
if(d.comment.length == 0 && skip_undocumented)
continue;
auto name = d.name.text;
descendInto(name);
auto comment = parseDocumentationComment(dittoSupport(d, d.comment), fullyQualifiedName ~ name);
auto app = appender!(char[])();
if (variableDeclaration.type !is null)
{
auto f = new MyFormatter!(typeof(app))(app);
f.format(variableDeclaration.type);
}
//output.putTag(app.data);
//output.put(" ");
//output.put(d.name.text);
//writeToParentList("variable " ~ cast(string)app.data ~ " ", name, comment.synopsis, "variable");
ascendOutOf(name);
}
}
override void visit(const Unittest ut) {
// skipping, we don't care about decls inside
}
}
/*
So as we descend, we need to be able to output stuff to a summary file,
a details file, and maybe a sitemap.
module test;
class Foo {
class Bar {
void wtf() {}
}
void cool() {}
}
That should generate these files:
test.html
test.Foo.html
test.Foo.Bar.html
test.Foo.Bar.wtf.1.html // the .1 is is a lame way of enabling overloads.
test.Foo.cool.html
test.html should contain the module comment and a list with Foo.
test.Foo.html should contain the class comment and a list with Bar and cool.
test.Foo.Bar.html should contain the Bar comment and a list with wtf.
So, on each level we descend, we open a new details file, which includes the
summary of its immediate children. Two file handles are needed:
File parent; // write basic info here
File current; // write details here
These are stacks so as we go up, we can pop them back off. When we descend,
current becomes parent and a new file for current is opened.
The sitemap has a recursive summary. It is a list of all the lists written out.
It will be done in a previous pass so we can do navigation upon opening a new
file too.
*/
class Documenter : ASTVisitor
{
alias visit = ASTVisitor.visit;
static struct MyOutputRange {
this(std.stdio.File output) {
this.output = output;
}
std.stdio.File output;
void put(T...)(T s) {
foreach(i; s)
putTag(i.htmlEncode);
}
void putTag(in char[] s) {
foreach(ch; s) {
assert(s);
assert(s.indexOf("
`
);
lastCommentChain ~= null;
overloadChain ~= null;
overloadNodeChain ~= null;
dittoChain ~= null;
}
void ascendOutOf(string name) {
assert(name == fullyQualifiedName[$-1]);
output.output.write(`
`);
auto us = "";
foreach(n; fullyQualifiedName[0 .. $-1]) {
if(us.length) us ~= ".";
us ~= n;
output.output.write("
"~n~"");
}
if(us.length)
us ~= ".";
us ~= fullyQualifiedName[$-1];
if(auto lol = join(fullyQualifiedName[0 .. $-1], ".") in additionalModuleInfo.childrenList) {
auto thing = (*lol).dup;
foreach(child; thing.sort()) {
auto disp = child;
auto i = disp.lastIndexOf(".");
if(i != -1)
disp = disp[i + 1 .. $];
auto cn = "";
if(us == child)
cn = "current";
if(isNumeric(disp))
continue; // skip overloads
output.output.write("
"~disp~"");
}
}
output.output.write(`
This doc is experimental and still undergoing lots of development.
Last updated:
`~__DATE__~`
") == -1);
}
output.write(s);
}
}
MyOutputRange output;
MyOutputRange[] parentChain;
string[] lastCommentChain;
int[string][] overloadChain; // FIXME: the doc needs to explain overload sets too
ASTNode[] dittoChain; // no Object ref ugh so casting
ASTNode[] overloadNodeChain;
string dittoSupport(const ASTNode node, string comment) {
auto c = preprocessComment(comment).toLower.strip;
if(lastCommentChain.length == 0) {
lastCommentChain ~= null;
dittoChain ~= null;
}
if(c == "ditto") {
return lastCommentChain[$-1];
} else if(c.length) {
dittoChain[$-1] = cast() node;
return (lastCommentChain[$-1] = comment);
}
else return comment;
}
string[] fullyQualifiedName;
void descendInto(string name) {
assert(name.length);
// this really means "if output is open"
if(fullyQualifiedName.length) {
parentChain ~= output;
}
fullyQualifiedName ~= name;
this.output = MyOutputRange(File(getFileName(), "wt"));
output.output.write(`