Implement partial include support

dstep/Configuration.d

@@ -113,6 +113,9 @@ struct Configuration
     @("public-global-import", "Add <import> as a public global import.")
     string[] publicGlobalImports;
 
+    /// needed for converting absolute includes to relative
+    string[] includePaths = ["/usr/local/include", "/usr/include"];
+
     Options toOptions(string inputFile, string outputFile) const
     {
         Options options = toOptions();
@@ -150,6 +153,7 @@ struct Configuration
         options.globalAttributes = globalAttributes;
         options.globalImports = globalImports;
         options.publicGlobalImports = publicGlobalImports;
+        options.includePaths = includePaths;
 
         return options;
     }

dstep/driver/CommandLine.d

@@ -8,6 +8,8 @@ module dstep.driver.CommandLine;
 import std.typecons : tuple, Tuple;
 import std.getopt;
 
+import clang.Util : asAbsNormPath;
+
 import dstep.Configuration;
 import dstep.translator.Options;
 import dstep.core.Exceptions;
@@ -86,7 +88,13 @@ auto parseCommandLine(string[] args)
 
     // Post-processing of CLI
 
-    import std.algorithm : canFind;
+    import std.algorithm;
+    import std.array : array;
+
+    config.includePaths ~= config.clangParams.filter!(p => p.length > 2 && p.startsWith("-I"))
+                                             .map!(p => p[2 .. $].asAbsNormPath).array;
+
+    config.includePaths = config.includePaths.sort().uniq.array.sort!((a, b) => a.length > b.length).array;
 
     if (forceObjectiveC)
         config.clangParams ~= "-ObjC";

dstep/translator/Context.d

@@ -11,13 +11,15 @@ import clang.Cursor;
 import clang.Index;
 import clang.SourceRange;
 import clang.TranslationUnit;
+import clang.Util;
 
 import dstep.translator.CommentIndex;
 import dstep.translator.IncludeHandler;
 import dstep.translator.MacroDefinition;
 import dstep.translator.MacroIndex;
 import dstep.translator.Options;
 import dstep.translator.Output;
+import dstep.translator.Preprocessor;
 import dstep.translator.Translator;
 import dstep.translator.TypedefIndex;
 import dstep.translator.HeaderIndex;
@@ -46,6 +48,8 @@ class Context
 
     public this(TranslationUnit translUnit, Options options, Translator translator = null)
     {
+        import std.algorithm.iteration : filter;
+
         this.translUnit = translUnit;
         macroIndex = new MacroIndex(translUnit);
         includeHandler_ = new IncludeHandler(headerIndex, options);
@@ -56,6 +60,14 @@ class Context
         foreach (item; options.publicGlobalImports)
             includeHandler_.addImport(item, Visibility.public_);
 
+        foreach (cursor; translUnit.cursor.children.filter!(c => c.kind == CXCursorKind.inclusionDirective))
+        {
+            if (cursor.path.asAbsNormPath != translUnit.spelling.asAbsNormPath)
+                continue;
+
+            includeHandler_.addInclude(cursor.spelling, cursor.includedFile.absolutePath);
+        }
+
         this.options = options;
 
         if (options.enableComments)

dstep/translator/HeaderIndex.d

@@ -225,6 +225,7 @@ class HeaderIndex
     private IncludeGraph includeGraph_;
     private string[string] stdLibPaths;
     private string[string] knownModules;
+    private string[string] includeNames;
     private string mainFilePath;
 
     public this(TranslationUnit translationUnit)
@@ -318,6 +319,11 @@ class HeaderIndex
 
     private this(T)(T directives, const string[string] moduleMapping)
     {
+        foreach (directive; directives)
+        {
+            includeNames.require(directive.includedFile.absolutePath, directive.spelling);
+        }
+
         knownModules = resolveKnownModules(
             directives,
             resolveKnownModulePaths(
@@ -331,6 +337,11 @@ class HeaderIndex
         return knownModule !is null ? *knownModule : null;
     }
 
+    string getIncludeName(string absolutePath)
+    {
+        return includeNames.get(absolutePath, null);
+    }
+
     IncludeGraph includeGraph()
     {
         return includeGraph_;

dstep/translator/IncludeHandler.d

@@ -25,7 +25,7 @@ class IncludeHandler
 {
     private Options options;
     private string[string] submodules;
-    private bool[string] includes;
+    private string[string] includes;
     private bool[string] imports;
     private Set!string publicImports;
     private HeaderIndex headerIndex;
@@ -106,6 +106,7 @@ class IncludeHandler
             "sys/un" : "core.sys.posix.sys.un",
             "sys/utsname" : "core.sys.posix.sys.utsname",
             "sys/wait" : "core.sys.posix.sys.wait",
+            "sys/ioctl" : "core.sys.posix.sys.ioctl",
 
             "windows" : "core.sys.windows.windows"
         ];
@@ -133,21 +134,40 @@ class IncludeHandler
         }
     }
 
-    void addInclude (string include)
+    void addInclude (string name, string absolutePath)
     {
-        import std.path;
-        import std.file;
-        import std.array;
+        assert(name.length > 0);
+        assert(absolutePath.length > 0);
 
-        auto absolute = include.asAbsNormPath;
+        includes.require(absolutePath, name);
+    }
+
+    void addInclude (string absolutePath)
+    {
+        import std.path : isAbsolute, baseName;
+        import std.algorithm.searching : startsWith;
 
-        if (absolute != options.inputFile && !include.empty)
+        auto includeName = headerIndex.getIncludeName(absolutePath);
+        assert(includeName.length > 0);
+
+        if (isAbsolute(includeName))
         {
-            if (exists(absolute) && isFile(absolute))
-                includes[absolute] = true;
-            else
-                includes[include] = true;
+            foreach (includePath; options.includePaths)
+            {
+                if (absolutePath.startsWith(includePath))
+                {
+                    includeName = absolutePath[includePath.length + 1 .. $];
+                    break;
+                }
+            }
+
+
+            if (isAbsolute(includeName))
+                // didn't match any known path, fallback to just filename
+                includeName = absolutePath.baseName;
         }
+
+        addInclude(includeName, absolutePath);
     }
 
     void addImport (string imp)
@@ -167,36 +187,48 @@ class IncludeHandler
 
     void addCompatible ()
     {
-        includes["config.h"] = true;
+        includes["config.h"] = "config.h";
     }
 
     void toImports (Output output)
     {
-        import std.algorithm : map;
-        import std.array : array;
+        import std.algorithm : map, sort, copy;
+        import std.array : array, replace;
         import std.format : format;
-        import std.algorithm.iteration : filter, map;
+        import std.algorithm.iteration : map, uniq;
+        import std.path : stripExtension;
+        import std.uni : toLower;
 
         Set!string standard, package_, unhandled;
 
-        foreach (entry; includes.byKey)
+        foreach (absolute, includeName; includes)
         {
-            if (auto i = isKnownInclude(entry))
-                standard.add(toImport(i));
-            else if (auto i = isPackageSubmodule(entry))
+            if (auto i = isPackageSubmodule(absolute))
                 package_.add(toSubmoduleImport(i));
-            else
-                unhandled.add(format(`/+ #include "%s" +/`, entry));
+            else if (includeName.length > 0)
+            {
+                if (auto i = isKnownInclude(includeName))
+                {
+                    standard.add(toImport(i));
+                }
+                else
+                {
+                    auto name = stripExtension(includeName).toLower();
+                    unhandled.add(format(`// FIXME: import %s;`, name.replace("/", ".")));
+                }
+            }
         }
 
         const publicExtra = publicImports.byKey.map!(e => toImport(e, Visibility.public_)).array;
         auto extra = imports.byKey.map!(e => toImport(e)).array;
+        auto imports = standard.keys ~ publicExtra ~ extra.array;
+        imports.sort();
+        imports.length -= imports.uniq().copy(imports).length;
 
-        importsBlock(output, standard.keys ~ publicExtra ~ extra.array);
+        importsBlock(output, imports);
         importsBlock(output, package_.keys);
 
-        if (options.keepUntranslatable)
-            importsBlock(output, unhandled.keys);
+        importsBlock(output, unhandled.keys);
 
         output.finalize();
     }
@@ -248,11 +280,11 @@ private:
 
     string isKnownInclude (string include)
     {
-        import std.path : stripExtension, baseName;
+        import std.path : stripExtension;
 
-        include = stripExtension(baseName(include));
+        auto name = stripExtension(include);
 
-        if (auto ptr = include in knownIncludes)
+        if (auto ptr = name in knownIncludes)
             return *ptr;
         else
             return null;

dstep/translator/Options.d

@@ -35,7 +35,6 @@ struct Options
     bool enableComments = true;
     bool publicSubmodules = false;
     bool normalizeModules = false;
-    bool keepUntranslatable = false;
     bool reduceAliases = true;
     bool translateMacros = true;
     bool portableWCharT = true;
@@ -51,6 +50,7 @@ struct Options
     const(string)[] globalAttributes;
     const(string)[] globalImports;
     const(string)[] publicGlobalImports;
+    const(string)[] includePaths;
     bool delegate(ref const(Cursor)) isWantedCursorForTypedefs;
 
     string toString() const

dstep/translator/Preprocessor.d

@@ -70,6 +70,12 @@ class PragmaDirective : Directive
 {
 }
 
+class IncludeDirective : Directive
+{
+    bool system;
+    string path;
+}
+
 class DefineDirective : Directive
 {
     MacroDefinition macroDefinition;
@@ -458,6 +464,30 @@ struct DirectiveRange
 
     Directive parseInclude(Token[] tokens)
     {
+        import std.algorithm.searching;
+        import std.algorithm.iteration;
+        import std.string : strip;
+
+        if (acceptDirective!"include"(tokens))
+        {
+            auto directive = new IncludeDirective();
+            directive.kind = DirectiveKind.include;
+            directive.tokens = tokens;
+            directive.extent = tokens.extent;
+
+            if (acceptPunctuation!("<")(tokens))
+            {
+                directive.system = true;
+                directive.path = tokens.until!((token) => token.spelling == ">")
+                                       .fold!((str, token) => str ~= token.spelling)("");
+            } else if (tokens.length == 1)
+            {
+                directive.system = false;
+                directive.path = tokens[0].spelling.strip("\"");
+            }
+            return directive;
+        }
+
         return null;
     }
 
@@ -1006,3 +1036,71 @@ unittest
 
     assert(case0[0].length == 3);
 }
+
+// System include directive.
+unittest
+{
+    auto case0 = directives(q"C
+#include <stdio.h>
+int x;
+C");
+
+    assert(case0.length == 1);
+    assert(case0[0].kind == DirectiveKind.include);
+
+    auto include = cast(IncludeDirective) case0[0];
+    assert(include !is null);
+    assert(include.system == true);
+    assert(include.path == "stdio.h");
+}
+
+// Local include directive.
+unittest
+{
+    auto case0 = directives(q"C
+#include "local.h"
+int x;
+C");
+
+    assert(case0.length == 1);
+    assert(case0[0].kind == DirectiveKind.include);
+
+    auto include = cast(IncludeDirective) case0[0];
+    assert(include !is null);
+    assert(include.system == false);
+    assert(include.path == "local.h");
+}
+
+// Multiple include directives.
+unittest
+{
+    auto case0 = directives(q"C
+#include <stdio.h>
+#include "local.h"
+#include <stdlib.h>
+#include "local2.h"
+int x;
+C");
+
+    assert(case0.length == 4);
+
+    auto include0 = cast(IncludeDirective) case0[0];
+    assert(include0 !is null);
+    assert(include0.system == true);
+    assert(include0.path == "stdio.h");
+
+    auto include1 = cast(IncludeDirective) case0[1];
+    assert(include1 !is null);
+    assert(include1.system == false);
+    assert(include1.path == "local.h");
+
+    auto include2 = cast(IncludeDirective) case0[2];
+    assert(include2 !is null);
+    assert(include2.system == true);
+    assert(include2.path == "stdlib.h");
+
+    auto include3 = cast(IncludeDirective) case0[3];
+    assert(include3 !is null);
+    assert(include3.system == false);
+    assert(include3.path == "local2.h");
+}