Add capabilities for merging CSDLs (#155)

* Update to the latest markdowndeep drop

* Add capability to merge EntityFramework objects together

* Updates to merge logic

* Fix up bindingParameter/this confusion.

* Improvements for publishing schema

* Updated dedupe logic

* Fix typo

* Handle some additional docs scenarios

* Make orphan pages an option

Author: rgregg

ApiDocs.Console/CommandLineOptions.cs

@@ -49,12 +49,12 @@ class CommandLineOptions
         public const string VerbPublishMetadata = "publish-edmx";
 
         public const string VerbGenerateDocs = "generate-docs";
-        
-        [VerbOption(VerbPrint, HelpText="Print files, resources, and methods discovered in the documentation.")]
+
+        [VerbOption(VerbPrint, HelpText = "Print files, resources, and methods discovered in the documentation.")]
         public PrintOptions PrintVerbOptions { get; set; }
 
         [VerbOption(VerbCheckLinks, HelpText = "Verify links in the documentation aren't broken.")]
-        public BasicCheckOptions CheckLinksVerb { get; set; }
+        public CheckLinkOptions CheckLinksVerb { get; set; }
 
         [VerbOption(VerbDocs, HelpText = "Check for errors in the documentation (resources + examples).")]
         public BasicCheckOptions CheckDocsVerb { get; set; }
@@ -65,19 +65,19 @@ class CommandLineOptions
         [VerbOption(VerbService, HelpText = "Check for errors between the documentation and service.")]
         public CheckServiceOptions CheckServiceVerb { get; set; }
 
-        [VerbOption(VerbPublish, HelpText="Publish a version of the documentation, optionally converting it into other formats.")]
+        [VerbOption(VerbPublish, HelpText = "Publish a version of the documentation, optionally converting it into other formats.")]
         public PublishOptions PublishVerb { get; set; }
 
-        [VerbOption(VerbPublishMetadata, HelpText="Publish or update metadata based on information in the docset.")]
+        [VerbOption(VerbPublishMetadata, HelpText = "Publish or update metadata based on information in the docset.")]
         public PublishMetadataOptions EdmxPublishVerb { get; set; }
 
-        [VerbOption(VerbMetadata, HelpText="Check service CSDL metadata against documentation.")]
+        [VerbOption(VerbMetadata, HelpText = "Check service CSDL metadata against documentation.")]
         public CheckMetadataOptions CheckMetadataVerb { get; set; }
 
-        [VerbOption(VerbGenerateDocs, HelpText="Generate documentation from an CSDL model")]
+        [VerbOption(VerbGenerateDocs, HelpText = "Generate documentation from an CSDL model")]
         public GenerateDocsOptions GenerateDocsVerb { get; set; }
 
-        [VerbOption(VerbAbout, HelpText="Print about information for this application.")]
+        [VerbOption(VerbAbout, HelpText = "Print about information for this application.")]
         public BaseOptions AboutVerb { get; set; }
 
         [HelpVerbOption]
@@ -90,7 +90,7 @@ public string GetUsage(string verb)
     class BaseOptions
     {
 
-        [Option("log", HelpText="Write the console output to file.")]
+        [Option("log", HelpText = "Write the console output to file.")]
         public string LogFile { get; set; }
 
         [Option("ignore-warnings", HelpText = "Ignore warnings as errors for pass rate.")]
@@ -99,10 +99,10 @@ class BaseOptions
         [Option("silence-warnings", HelpText = "Don't print warnings to the screen or consider them errors")]
         public bool SilenceWarnings { get; set; }
 
-        [Option("appveyor-url", HelpText="Specify the AppVeyor Build Worker API URL for output integration")]
+        [Option("appveyor-url", HelpText = "Specify the AppVeyor Build Worker API URL for output integration")]
         public string AppVeyorServiceUrl { get; set; }
 
-        [Option("ignore-errors", HelpText="Prevent errors from generating a non-zero return code.")]
+        [Option("ignore-errors", HelpText = "Prevent errors from generating a non-zero return code.")]
         public bool IgnoreErrors { get; set; }
 
         [Option("parameters", HelpText = "Specify additional page variables that are used by the publishing engine. URL encoded: key=value&key2=value2.")]
@@ -131,7 +131,7 @@ public Dictionary<string, string> PageParameterDict {
         }
 
 #if DEBUG
-        [Option("debug", HelpText="Launch the debugger before doing anything interesting")]
+        [Option("debug", HelpText = "Launch the debugger before doing anything interesting")]
         public bool AttachDebugger { get; set; }
 #endif
 
@@ -183,16 +183,16 @@ class CheckMetadataOptions : DocSetOptions
 
     class PrintOptions : DocSetOptions
     {
-        [Option("files", HelpText="Print the files discovered as part of the documentation")]
+        [Option("files", HelpText = "Print the files discovered as part of the documentation")]
         public bool PrintFiles { get; set; }
-        
-        [Option("resources", HelpText="Print the resources discovered in the documentation")]
+
+        [Option("resources", HelpText = "Print the resources discovered in the documentation")]
         public bool PrintResources { get; set; }
-        
-        [Option("methods", HelpText="Print the methods discovered in the documentation.")]
+
+        [Option("methods", HelpText = "Print the methods discovered in the documentation.")]
         public bool PrintMethods { get; set; }
 
-        [Option("accounts", HelpText="Print the list of accounts discovered in the documentation.")]
+        [Option("accounts", HelpText = "Print the list of accounts discovered in the documentation.")]
         public bool PrintAccounts { get; set; }
 
         public override bool HasRequiredProperties(out string[] missingArguments)
@@ -212,6 +212,11 @@ public override bool HasRequiredProperties(out string[] missingArguments)
         }
     }
 
+    class CheckLinkOptions : BasicCheckOptions {
+        [Option("orphan-page-warning", HelpText="Print a warning for each page without any incoming links.")]
+        public bool IncludeOrphanPageWarning { get; set; }
+    }
+
     class BasicCheckOptions : DocSetOptions
     {
         [Option('m', "method", HelpText = "Name of the method to test. If omitted, all defined methods are tested.", MutuallyExclusiveSet="fileOrMethod")]
@@ -392,6 +397,9 @@ class PublishMetadataOptions : DocSetOptions
         [Option("source", HelpText="Source metadata input file.")]
         public string SourceMetadataPath { get; set; }
 
+        [Option("merge-with", HelpText= "Specify a second metadata input file to merge with the first.")]
+        public string SecondSourceMetadataPath { get; set; }
+
         [Option("format", DefaultValue=MetadataFormat.Default, HelpText="Specify the input and output formats for metadata.")]
         public MetadataFormat DataFormat { get; set; }
 
@@ -428,6 +436,7 @@ public CsdlWriterOptions GetOptions()
                 Sort = SortOutput,
                 OutputDirectoryPath = OutputDirectory,
                 SourceMetadataPath = SourceMetadataPath,
+                MergeWithMetadataPath = SecondSourceMetadataPath,
                 Namespaces = Namespaces?.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries),
                 TransformOutput = TransformOutput,
                 DocumentationSetPath = DocumentationSetPath,

ApiDocs.Console/Program.cs

@@ -181,13 +181,13 @@ private static async Task RunInvokedMethodAsync(CommandLineOptions origCommandLi
                     await PrintDocInformationAsync((PrintOptions)options);
                     break;
                 case CommandLineOptions.VerbCheckLinks:
-                    returnSuccess = await CheckLinksAsync((BasicCheckOptions)options);
+                    returnSuccess = await CheckLinksAsync((CheckLinkOptions)options);
                     break;
                 case CommandLineOptions.VerbDocs:
                     returnSuccess = await CheckDocsAsync((BasicCheckOptions)options);
                     break;
                 case CommandLineOptions.VerbCheckAll:
-                    returnSuccess = await CheckDocsAllAsync((BasicCheckOptions)options);
+                    returnSuccess = await CheckDocsAllAsync((CheckLinkOptions)options);
                     break;
                 case CommandLineOptions.VerbService:
                     returnSuccess = await CheckServiceAsync((CheckServiceOptions)options);
@@ -220,7 +220,7 @@ private static async Task RunInvokedMethodAsync(CommandLineOptions origCommandLi
         /// </summary>
         /// <param name="options"></param>
         /// <returns></returns>
-        private static async Task<bool> CheckDocsAllAsync(BasicCheckOptions options)
+        private static async Task<bool> CheckDocsAllAsync(CheckLinkOptions options)
         {
             var docset = await GetDocSetAsync(options);
 
@@ -468,7 +468,7 @@ private static async Task PrintAccountsAsync(PrintOptions options, DocSet docset
         /// </summary>
         /// <param name="options"></param>
         /// <param name="docs"></param>
-        private static async Task<bool> CheckLinksAsync(BasicCheckOptions options, DocSet docs = null)
+        private static async Task<bool> CheckLinksAsync(CheckLinkOptions options, DocSet docs = null)
         {
             const string testName = "Check-links";
             var docset = docs ?? await GetDocSetAsync(options);
@@ -487,7 +487,7 @@ private static async Task<bool> CheckLinksAsync(BasicCheckOptions options, DocSe
             TestReport.StartTest(testName);
 
             ValidationError[] errors;
-            docset.ValidateLinks(options.EnableVerboseOutput, interestingFiles, out errors, options.RequireFilenameCaseMatch);
+            docset.ValidateLinks(options.EnableVerboseOutput, interestingFiles, out errors, options.RequireFilenameCaseMatch, options.IncludeOrphanPageWarning);
 
             foreach (var error in errors)
             {
@@ -616,7 +616,7 @@ private static async Task<CheckResults> CheckExamplesAsync(BasicCheckOptions opt
                     if (example.Language != CodeLanguage.Json)
                         continue;
 
-                    var testName = string.Format("check-example: {0}", example.Metadata.MethodName, example.Metadata.ResourceType);
+                    var testName = string.Format("check-example: {0}", example.Metadata.MethodName?.FirstOrDefault(), example.Metadata.ResourceType);
                     TestReport.StartTest(testName, doc.DisplayName);
 
                     ValidationError[] errors;
@@ -650,7 +650,7 @@ private static async Task<CheckResults> CheckMethodsAsync(BasicCheckOptions opti
 
                 if (string.IsNullOrEmpty(method.ExpectedResponse))
                 {
-                    await TestReport.FinishTestAsync(testName, TestOutcome.Failed, "Null response where one was expected.", printFailuresOnly: options.PrintFailuresOnly);
+                    await TestReport.FinishTestAsync(testName, TestOutcome.Failed, "No response was paired with this request.", printFailuresOnly: options.PrintFailuresOnly);
                     results.FailureCount++;
                     continue;
                 }

ApiDocs.Publishing/ApiDocs.Publishing.csproj

@@ -49,6 +49,7 @@
   <ItemGroup>
     <Compile Include="CSDL\CsdlExtensionMethods.cs" />
     <Compile Include="CSDL\CsdlWriter.cs" />
+    <Compile Include="CSDL\ObjectGraphMerger.cs" />
     <Compile Include="CSDL\MethodCollection.cs" />
     <Compile Include="Html\ApiDocsConditionalTag.cs" />
     <Compile Include="Html\DocumentPublisherHtml.cs" />

ApiDocs.Publishing/CSDL/CsdlExtensionMethods.cs

@@ -27,7 +27,15 @@ namespace ApiDocs.Publishing.CSDL
 {
     using ApiDocs.Validation;
     using System;
+    using System.Collections.Generic;
+    using System.Reflection;
+    using System.Text;
     using Validation.Http;
+    using Validation.OData;
+    using Validation.OData.Transformation;
+    using Validation.Utility;
+    using System.Linq;
+
     internal static class CsdlExtensionMethods
     {
 
@@ -67,5 +75,43 @@ internal static void AppendWithCondition(this System.Text.StringBuilder sb, bool
                 sb.Append(text);
             }
         }
+
+        /// <summary>
+        /// Merge two EntityFramework instances together into the first framework
+        /// </summary>
+        /// <param name="framework1"></param>
+        /// <param name="framework2"></param>
+        internal static EntityFramework MergeWith(this EntityFramework framework1, EntityFramework framework2)
+        {
+            ObjectGraphMerger<EntityFramework> merger = new ObjectGraphMerger<EntityFramework>(framework1, framework2);
+            var edmx = merger.Merge();
+
+            // Clean up bindingParameters on actions and methods to be consistently the same
+            foreach(var schema in edmx.DataServices.Schemas)
+            {
+                foreach (var action in schema.Actions)
+                {
+                    foreach(var param in action.Parameters.Where(x => x.Name == "bindingParameter" || x.Name == "this")) {
+                        param.Name = "bindingParameter";
+                        param.Nullable = null;
+                    }
+                }
+                foreach(var func in schema.Functions)
+                {
+                    foreach (var param in func.Parameters.Where(x => x.Name == "bindingParameter" || x.Name == "this")) {
+                        param.Name = "bindingParameter";
+                        param.Nullable = null;
+                    }
+                }
+            }
+
+            return edmx;
+
+
+        }
+
+       
+
+       
     }
 }

ApiDocs.Publishing/CSDL/CsdlWriter.cs

@@ -61,13 +61,20 @@ public CsdlWriter(DocSet docs, CsdlWriterOptions options)
 
         public override async Task PublishToFolderAsync(string outputFolder)
         {
-            string outputFilenameSuffix = null;
+            string outputFilenameSuffix = "";
 
             // Step 1: Generate an EntityFramework OM from the documentation and/or template file
             EntityFramework framework = CreateEntityFrameworkFromDocs();
             if (null == framework)
                 return;
 
+            if (!string.IsNullOrEmpty(options.MergeWithMetadataPath))
+            {
+                EntityFramework secondFramework = CreateEntityFrameworkFromDocs(options.MergeWithMetadataPath, generateFromDocs: false);
+                framework = framework.MergeWith(secondFramework);
+                outputFilenameSuffix += "-merged";
+            }
+
             // Step 1a: Apply an transformations that may be defined in the documentation
             if (!string.IsNullOrEmpty(options.TransformOutput))
             {
@@ -81,7 +88,7 @@ public override async Task PublishToFolderAsync(string outputFolder)
                 framework.ApplyTransformation(transformations.SchemaChanges, versionsToPublish);
                 if (!string.IsNullOrEmpty(options.Version))
                 {
-                    outputFilenameSuffix = $"-{options.Version}";
+                    outputFilenameSuffix += $"-{options.Version}";
                 }
             }
 
@@ -141,19 +148,21 @@ private string GenerateOutputFileFullName(string templateFilename, string output
             return outputFullName;
         }
 
-        private EntityFramework CreateEntityFrameworkFromDocs()
+        private EntityFramework CreateEntityFrameworkFromDocs(string sourcePath = null, bool? generateFromDocs = null)
         {
+            sourcePath = sourcePath ?? options.SourceMetadataPath;
+
             EntityFramework edmx = new EntityFramework();
-            if (!string.IsNullOrEmpty(options.SourceMetadataPath))
+            if (!string.IsNullOrEmpty(sourcePath))
             {
                 try
                 {
-                    if (!System.IO.File.Exists(options.SourceMetadataPath))
+                    if (!System.IO.File.Exists(sourcePath))
                     {
-                        throw new System.IO.FileNotFoundException($"Unable to locate source file: {options.SourceMetadataPath}");
+                        throw new System.IO.FileNotFoundException($"Unable to locate source file: {sourcePath}");
                     }
 
-                    using (System.IO.FileStream stream = new System.IO.FileStream(options.SourceMetadataPath, System.IO.FileMode.Open, System.IO.FileAccess.Read))
+                    using (System.IO.FileStream stream = new System.IO.FileStream(sourcePath, System.IO.FileMode.Open, System.IO.FileAccess.Read))
                     {
                         if (options.Formats.HasFlag(MetadataFormat.EdmxInput))
                         {
@@ -193,10 +202,10 @@ private EntityFramework CreateEntityFrameworkFromDocs()
                 }
             }
 
-            bool generateNewElements = !options.SkipMetadataGeneration;
+            bool generateNewElements = (generateFromDocs == null && !options.SkipMetadataGeneration) || (generateFromDocs.HasValue && generateFromDocs.Value) ;
 
             // Add resources
-            if (Documents.Files.Any())
+            if (generateNewElements && Documents.Files.Any())
             {
                 foreach (var resource in Documents.Resources)
                 {
@@ -523,7 +532,7 @@ private Dictionary<string, MethodCollection> GetUniqueRequestPaths(string baseUr
                 Dictionary<string, MethodCollection> uniqueRequestPaths = new Dictionary<string, MethodCollection>();
                 foreach (var m in Documents.Methods)
                 {
-                    if (m.ExpectedResponseMetadata.ExpectError)
+                    if (m.ExpectedResponseMetadata != null && m.ExpectedResponseMetadata.ExpectError)
                     {
                         // Ignore thigns that are expected to error
                         continue;
@@ -544,7 +553,7 @@ private Dictionary<string, MethodCollection> GetUniqueRequestPaths(string baseUr
                     }
                     uniqueRequestPaths[path].Add(m);
 
-                    Console.WriteLine("{0} :: {1} --> {2}", path, m.RequestMetadata.ResourceType, m.ExpectedResponseMetadata.ResourceType);
+                    Console.WriteLine("{0} :: {1} --> {2}", path, m.RequestMetadata.ResourceType, m.ExpectedResponseMetadata?.ResourceType);
                 }
                 cachedUniqueRequestPaths = uniqueRequestPaths;
             }
@@ -792,6 +801,7 @@ public class CsdlWriterOptions
     {
         public string OutputDirectoryPath { get; set; }
         public string SourceMetadataPath { get; set; }
+        public string MergeWithMetadataPath { get; set; }
         public MetadataFormat Formats { get; set; }
         public string[] Namespaces { get; set; }
         public bool Sort { get; set; }

ApiDocs.Publishing/CSDL/ObjectGraphMerger.cs

@@ -0,0 +1 @@
+

ApiDocs.Validation.UnitTests/ApiDocs.Validation.UnitTests.csproj

@@ -53,6 +53,7 @@
     <Compile Include="JsonRewriteTests.cs" />
     <Compile Include="MultipartMimeTests.cs" />
     <Compile Include="NullableTests.cs" />
+    <Compile Include="ObjectGraphMergerTests.cs" />
     <Compile Include="Properties\Resources.Designer.cs">
       <AutoGen>True</AutoGen>
       <DesignTime>True</DesignTime>