From 4eb0999856e79b0b5e510d2b7bcbe6ee69528a6f Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Mon, 11 Jul 2016 17:16:34 -0400 Subject: [PATCH 01/17] Adding [TAGS] support - Moved PageParameters variable and GeneratePageParameters method from derived HtmlMustacheWriter class to base DocumentPublisherHtml class - Added TagProcessor class that does pre and post processing of the markdown and HTML - Moved code that removed double block quotes from PublishFileToDestinationAsync to TagProcessor.PreProcess --- ApiDocs.Publishing/ApiDocs.Publishing.csproj | 1 + .../Html/DocumentPublisherHtml.cs | 39 ++- ApiDocs.Publishing/Html/HtmlMustacheWriter.cs | 17 -- ApiDocs.Publishing/Tags/TagProcessor.cs | 281 ++++++++++++++++++ 4 files changed, 305 insertions(+), 33 deletions(-) create mode 100644 ApiDocs.Publishing/Tags/TagProcessor.cs diff --git a/ApiDocs.Publishing/ApiDocs.Publishing.csproj b/ApiDocs.Publishing/ApiDocs.Publishing.csproj index e13b6d8..be7d5e4 100644 --- a/ApiDocs.Publishing/ApiDocs.Publishing.csproj +++ b/ApiDocs.Publishing/ApiDocs.Publishing.csproj @@ -61,6 +61,7 @@ + diff --git a/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs b/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs index e8028b2..e5dcaae 100644 --- a/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs +++ b/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs @@ -36,6 +36,7 @@ namespace ApiDocs.Publishing.Html using ApiDocs.Validation.Writers; using MarkdownDeep; using Newtonsoft.Json; + using Tags; public class DocumentPublisherHtml : DocumentPublisher { @@ -43,6 +44,7 @@ public class DocumentPublisherHtml : DocumentPublisher public string HtmlOutputExtension { get; set; } public string TemplateHtmlFilename { get; set; } + protected Dictionary PageParameters { get; set; } /// /// Allow HTML tags in the markdown source to pass through to the converted HTML. This is considered @@ -56,6 +58,7 @@ public DocumentPublisherHtml(DocSet docs, IPublishOptions options) TemplateHtmlFilename = options.TemplateFilename ?? "template.htm"; HtmlOutputExtension = options.OutputExtension ?? ".htm"; EnableHtmlTagPassThrough = options.AllowUnsafeHtmlContentInMarkdown; + PageParameters = GeneratePageParameters(options); } /// @@ -185,6 +188,7 @@ protected static void SplitUrlPathAndBookmark(string input, out string url, out /// protected virtual Markdown GetMarkdownConverter() { + //var converter = new Markdown var converter = new Markdown { ExtraMode = true, @@ -226,24 +230,12 @@ protected override async Task PublishFileToDestinationAsync(FileInfo sourceFile, var destinationPath = this.GetPublishedFilePath(sourceFile, destinationRoot, HtmlOutputExtension); - StringWriter writer = new StringWriter(); - StreamReader reader = new StreamReader(sourceFile.OpenRead()); - - long lineNumber = 0; - string nextLine; - while ((nextLine = await reader.ReadLineAsync()) != null) - { - lineNumber++; - if (this.IsDoubleBlockQuote(nextLine)) - { - this.LogMessage(new ValidationMessage(string.Concat(sourceFile.Name, ":", lineNumber), "Removing DoubleBlockQuote")); - continue; - } - await writer.WriteLineAsync(nextLine); - } + // Create a tag processor + TagProcessor tagProcessor = new TagProcessor(PageParameters?["tags"]?.ToString(), LogMessage); var converter = this.GetMarkdownConverter(); - var html = converter.Transform(writer.ToString()); + var html = converter.Transform(await tagProcessor.Preprocess(sourceFile)); + html = await tagProcessor.PostProcess(html); var pageData = page.Annotation ?? new PageAnnotation(); if (string.IsNullOrEmpty(pageData.Title)) @@ -342,6 +334,21 @@ private string ParseDocumentIfStatement(string key, string containingFilePath) } + private static Dictionary GeneratePageParameters(IPublishOptions options) + { + if (string.IsNullOrEmpty(options.AdditionalPageParameters)) + return null; + + var data = new Dictionary(); + + var parameters = Validation.Http.HttpParser.ParseQueryString(options.AdditionalPageParameters); + foreach (var key in parameters.AllKeys) + { + data[key] = parameters[key]; + } + return data; + } + // ReSharper disable once ClassNeverInstantiated.Local class IfQueryData { diff --git a/ApiDocs.Publishing/Html/HtmlMustacheWriter.cs b/ApiDocs.Publishing/Html/HtmlMustacheWriter.cs index 2f098ea..67889d3 100644 --- a/ApiDocs.Publishing/Html/HtmlMustacheWriter.cs +++ b/ApiDocs.Publishing/Html/HtmlMustacheWriter.cs @@ -39,7 +39,6 @@ public class HtmlMustacheWriter : DocumentPublisherHtml { private Generator generator; private FileTagDefinition fileTag; - private Dictionary PageParameters { get; set; } public bool CollapseTocToActiveGroup { get; set; } @@ -47,7 +46,6 @@ public class HtmlMustacheWriter : DocumentPublisherHtml public HtmlMustacheWriter(DocSet docs, IPublishOptions options) : base(docs, options) { this.CollapseTocToActiveGroup = false; - this.PageParameters = GeneratePageParameters(options); } protected override void LoadTemplate() @@ -129,21 +127,6 @@ protected override async Task WriteAdditionalFilesAsync() } } - private static Dictionary GeneratePageParameters(IPublishOptions options) - { - if (string.IsNullOrEmpty(options.AdditionalPageParameters)) - return null; - - var data = new Dictionary(); - - var parameters = Validation.Http.HttpParser.ParseQueryString(options.AdditionalPageParameters); - foreach (var key in parameters.AllKeys) - { - data[key] = parameters[key]; - } - return data; - } - private class PageTemplateInput { public PageAnnotation Page { get; set; } diff --git a/ApiDocs.Publishing/Tags/TagProcessor.cs b/ApiDocs.Publishing/Tags/TagProcessor.cs new file mode 100644 index 0000000..1688d18 --- /dev/null +++ b/ApiDocs.Publishing/Tags/TagProcessor.cs @@ -0,0 +1,281 @@ +/* + * Markdown Scanner + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the ""Software""), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using ApiDocs.Validation.Error; +using System.Text.RegularExpressions; + +namespace ApiDocs.Publishing.Tags +{ + class TagProcessor + { + private string[] TagsToInclude = null; + private static string[] tagSeparators = { ",", " " }; + + private static Regex ValidTagFormat = new Regex(@"^\[TAGS=[-\.\w]+(?:,\s?[-\.\w]*)*\]", RegexOptions.IgnoreCase); + private static Regex GetTagList = new Regex(@"\[TAGS=([-\.,\s\w]+)\]", RegexOptions.IgnoreCase); + + private Action LogMessage = null; + + public TagProcessor(string tags, Action logMethod = null) + { + if (!string.IsNullOrEmpty(tags)) + { + TagsToInclude = tags.ToUpper().Split(TagProcessor.tagSeparators, + StringSplitOptions.RemoveEmptyEntries); + } + + // If not logging method supplied default to a no-op + LogMessage = logMethod ?? DefaultLogMessage; + } + + /// + /// Loads Markdown content from a file and removes unwanted content in preparation for passing to MarkdownDeep converter. + /// + /// The file containing the Markdown contents to preprocess. + /// The preprocessed contents of the file. + public async Task Preprocess(FileInfo sourceFile) + { + StringWriter writer = new StringWriter(); + StreamReader reader = new StreamReader(sourceFile.OpenRead()); + + long lineNumber = 0; + bool inTag = false; + bool dropTagContent = false; + string nextLine; + while ((nextLine = await reader.ReadLineAsync()) != null) + { + lineNumber++; + + // Check if this is an [END] marker + if (IsEndLine(nextLine)) + { + // We SHOULD be in a tag + if (inTag) + { + if (!dropTagContent) + { + await writer.WriteLineAsync(nextLine); + } + + // Reset tag state + inTag = false; + dropTagContent = false; + } + else + { + LogMessage(new ValidationError(ValidationErrorCode.MarkdownParserError, + string.Concat(sourceFile.Name, ":", lineNumber), "Unexpected [END] marker.")); + } + + continue; + } + + if (inTag && dropTagContent) + { + // Inside of a tag that shouldn't be included + LogMessage(new ValidationMessage(string.Concat(sourceFile.Name, ":", lineNumber), "Removing tagged content")); + continue; + } + + // Remove double blockquotes (">>") + if (IsDoubleBlockQuote(nextLine)) + { + LogMessage(new ValidationMessage(string.Concat(sourceFile.Name, ":", lineNumber), "Removing DoubleBlockQuote")); + continue; + } + + // Check if this is a [TAGS] marker + if (IsTagLine(nextLine, sourceFile.Name, lineNumber)) + { + if (inTag) + { + // Nested tags not allowed + LogMessage(new ValidationError(ValidationErrorCode.MarkdownParserError, + string.Concat(sourceFile.Name, ":", lineNumber), "Nested tags are not supported.")); + } + + string[] tags = GetTags(nextLine); + + LogMessage(new ValidationMessage(string.Concat(sourceFile.Name, ":", lineNumber), "Found TAGS line with {0}", string.Join(",", tags))); + + inTag = true; + dropTagContent = !TagsAreIncluded(tags); + if (dropTagContent) + { + LogMessage(new ValidationMessage(string.Concat(sourceFile.Name, ":", lineNumber), "{0} are not found in the specified tags to include, content will be dropped.", string.Join(",", tags))); + } + else + { + // Replace line with div + await writer.WriteLineAsync(nextLine); + } + + continue; + } + + await writer.WriteLineAsync(nextLine); + } + + if (inTag) + { + // If inTag is true, there was a missing [END] tag somewhere + LogMessage(new ValidationError(ValidationErrorCode.MarkdownParserError, + sourceFile.Name, "The file ended while still in a [TAGS] tag. All [TAGS] must be closed with an [END] tag.")); + } + + return writer.ToString(); + } + + /// + /// Loads HTML from MarkdownDeep conversion process and replaces tags with <div> markers. + /// + /// The HTML content returned from MarkdownDeep. + /// The postprocessed HTML content. + public async Task PostProcess(string html) + { + StringWriter writer = new StringWriter(); + StringReader reader = new StringReader(html); + + // Checks for closed tag and nesting were handled in preprocessing, + // so not repeating them here + + string nextLine; + while ((nextLine = await reader.ReadLineAsync()) != null) + { + // Replace with
+ if (IsConvertedTagLine(nextLine)) + { + string[] tags = GetTags(nextLine); + await writer.WriteLineAsync(GetDivMarker(tags)); + continue; + } + + // Replace with
+ if (IsConvertedEndLine(nextLine)) + { + await writer.WriteLineAsync(GetEndDivMarker()); + continue; + } + + await writer.WriteLineAsync(nextLine); + } + + return writer.ToString(); + } + + private bool IsDoubleBlockQuote(string text) + { + return text.StartsWith(">>") || text.StartsWith(" >>"); + } + + private bool IsTagLine(string text, string fileName, long lineNumber) + { + bool looksLikeTag = text.Trim().ToUpper().StartsWith("[TAGS="); + + if (!looksLikeTag) return false; + + // It looks like a tag, but is it legit? + if(!ValidTagFormat.IsMatch(text.Trim())) + { + LogMessage(new ValidationError(ValidationErrorCode.MarkdownParserError, + string.Concat(fileName, ":", lineNumber), "Invalid TAGS line detected, ignoring...")); + return false; + } + + return true; + } + + private string[] GetTags(string text) + { + Match m = GetTagList.Match(text.Trim()); + + if (m.Success && m.Groups.Count == 2) + { + return m.Groups[1].Value.Split(TagProcessor.tagSeparators, + StringSplitOptions.RemoveEmptyEntries); + } + + return null; + } + + private bool TagsAreIncluded(string[] tags) + { + if (TagsToInclude == null) + { + return false; + } + + foreach (string tag in tags) + { + // If any tag matches included tags, return true + if (TagsToInclude.Contains(tag)) + { + return true; + } + } + + return false; + } + + private bool IsEndLine(string text) + { + return text.Trim().ToUpper().Equals("[END]"); + } + + private bool IsConvertedTagLine(string text) + { + return ValidTagFormat.IsMatch(text.Replace("

", "").Replace("

", "")); + } + + private bool IsConvertedEndLine(string text) + { + return text.Equals("

[END]

"); + } + + private string GetDivMarker(string[] tags) + { + for (int i = 0; i < tags.Length; i++) + { + tags[i] = string.Format("content-{0}", tags[i].ToLower().Replace('.', '-')); + } + + return string.Format("
", string.Join(" ", tags)); + } + + private string GetEndDivMarker() + { + return "
"; + } + + private void DefaultLogMessage(ValidationError msg) + { + // Empty method + } + } +} From 3bcfcc49df7662ebea48b1eb6f9a831dff842003 Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Tue, 12 Jul 2016 14:44:32 -0400 Subject: [PATCH 02/17] Implemented include file functionality --- .../Html/DocumentPublisherHtml.cs | 2 +- ApiDocs.Publishing/Tags/TagProcessor.cs | 75 ++++++++++++++++++- 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs b/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs index e5dcaae..cc4908e 100644 --- a/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs +++ b/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs @@ -235,7 +235,7 @@ protected override async Task PublishFileToDestinationAsync(FileInfo sourceFile, var converter = this.GetMarkdownConverter(); var html = converter.Transform(await tagProcessor.Preprocess(sourceFile)); - html = await tagProcessor.PostProcess(html); + html = await tagProcessor.PostProcess(html, sourceFile, converter); var pageData = page.Annotation ?? new PageAnnotation(); if (string.IsNullOrEmpty(pageData.Title)) diff --git a/ApiDocs.Publishing/Tags/TagProcessor.cs b/ApiDocs.Publishing/Tags/TagProcessor.cs index 1688d18..61d1668 100644 --- a/ApiDocs.Publishing/Tags/TagProcessor.cs +++ b/ApiDocs.Publishing/Tags/TagProcessor.cs @@ -29,6 +29,7 @@ using System.Threading.Tasks; using ApiDocs.Validation.Error; using System.Text.RegularExpressions; +using MarkdownDeep; namespace ApiDocs.Publishing.Tags { @@ -39,6 +40,7 @@ class TagProcessor private static Regex ValidTagFormat = new Regex(@"^\[TAGS=[-\.\w]+(?:,\s?[-\.\w]*)*\]", RegexOptions.IgnoreCase); private static Regex GetTagList = new Regex(@"\[TAGS=([-\.,\s\w]+)\]", RegexOptions.IgnoreCase); + private static Regex ConvertedIncludeFormat = new Regex(@"

\[INCLUDE\s*[-.\w]+\]

", RegexOptions.IgnoreCase); private Action LogMessage = null; @@ -156,8 +158,10 @@ public async Task Preprocess(FileInfo sourceFile) /// Loads HTML from MarkdownDeep conversion process and replaces tags with <div> markers. ///
/// The HTML content returned from MarkdownDeep. + /// The original Markdown file. + /// The Markdown object to use for converting include files. /// The postprocessed HTML content. - public async Task PostProcess(string html) + public async Task PostProcess(string html, FileInfo sourceFile, Markdown converter) { StringWriter writer = new StringWriter(); StringReader reader = new StringReader(html); @@ -183,6 +187,36 @@ public async Task PostProcess(string html) continue; } + // Load includes + if (IsConvertedIncludeLine(nextLine)) + { + FileInfo includeFile = GetIncludeFile(nextLine, sourceFile); + if (!includeFile.Exists) + { + LogMessage(new ValidationError(ValidationErrorCode.ErrorOpeningFile, nextLine, "The included file {0} was not found", includeFile.FullName)); + continue; + } + + if (includeFile != null) + { + if (includeFile.FullName.Equals(sourceFile.FullName)) + { + LogMessage(new ValidationError(ValidationErrorCode.MarkdownParserError, nextLine, "A Markdown file cannot include itself")); + continue; + } + + string includeContent = await GetIncludedContent(includeFile, converter); + + await writer.WriteLineAsync(includeContent); + } + else + { + LogMessage(new ValidationError(ValidationErrorCode.ErrorReadingFile, nextLine, "Could not load include content from {0}", includeFile.FullName)); + } + + continue; + } + await writer.WriteLineAsync(nextLine); } @@ -250,6 +284,13 @@ private bool IsEndLine(string text) private bool IsConvertedTagLine(string text) { + // To handle edge case where you have a [TAGS] type entry inside + // a code block. + if (!text.StartsWith("

")) + { + return false; + } + return ValidTagFormat.IsMatch(text.Replace("

", "").Replace("

", "")); } @@ -258,6 +299,30 @@ private bool IsConvertedEndLine(string text) return text.Equals("

[END]

"); } + private bool IsConvertedIncludeLine(string text) + { + if (text.ToUpper().Contains("INCLUDE")) + { + return ConvertedIncludeFormat.IsMatch(text); + } + + return false; + } + + private FileInfo GetIncludeFile(string text, FileInfo sourceFile) + { + Match m = ConvertedIncludeFormat.Match(text); + + if (m.Success && m.Groups.Count == 2) + { + string relativePath = Path.ChangeExtension(m.Groups[1].Value, "md"); + + return new FileInfo(Path.Combine(sourceFile.Directory.FullName, relativePath)); + } + + return null; + } + private string GetDivMarker(string[] tags) { for (int i = 0; i < tags.Length; i++) @@ -273,6 +338,14 @@ private string GetEndDivMarker() return ""; } + private async Task GetIncludedContent (FileInfo includeFile, Markdown converter) + { + // Do inline conversion, including pre and post processing + string html = converter.Transform(await Preprocess(includeFile)); + + return await PostProcess(html, includeFile, converter); + } + private void DefaultLogMessage(ValidationError msg) { // Empty method From caa979b7f7e2d7011241ec2a8f62ea834ec0a91d Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Tue, 12 Jul 2016 15:35:25 -0400 Subject: [PATCH 03/17] Added support for nested tagging --- ApiDocs.Publishing/Tags/TagProcessor.cs | 82 +++++++++++++++---------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/ApiDocs.Publishing/Tags/TagProcessor.cs b/ApiDocs.Publishing/Tags/TagProcessor.cs index 61d1668..5153a4b 100644 --- a/ApiDocs.Publishing/Tags/TagProcessor.cs +++ b/ApiDocs.Publishing/Tags/TagProcessor.cs @@ -67,8 +67,8 @@ public async Task Preprocess(FileInfo sourceFile) StreamReader reader = new StreamReader(sourceFile.OpenRead()); long lineNumber = 0; - bool inTag = false; - bool dropTagContent = false; + int tagCount = 0; + int dropCount = 0; string nextLine; while ((nextLine = await reader.ReadLineAsync()) != null) { @@ -78,16 +78,19 @@ public async Task Preprocess(FileInfo sourceFile) if (IsEndLine(nextLine)) { // We SHOULD be in a tag - if (inTag) + if (tagCount > 0) { - if (!dropTagContent) + if (dropCount <= 0) { await writer.WriteLineAsync(nextLine); } + else + { + dropCount--; + } - // Reset tag state - inTag = false; - dropTagContent = false; + // Decrement tag count + tagCount--; } else { @@ -98,53 +101,66 @@ public async Task Preprocess(FileInfo sourceFile) continue; } - if (inTag && dropTagContent) - { - // Inside of a tag that shouldn't be included - LogMessage(new ValidationMessage(string.Concat(sourceFile.Name, ":", lineNumber), "Removing tagged content")); - continue; - } - - // Remove double blockquotes (">>") - if (IsDoubleBlockQuote(nextLine)) - { - LogMessage(new ValidationMessage(string.Concat(sourceFile.Name, ":", lineNumber), "Removing DoubleBlockQuote")); - continue; - } - // Check if this is a [TAGS] marker if (IsTagLine(nextLine, sourceFile.Name, lineNumber)) { - if (inTag) - { - // Nested tags not allowed - LogMessage(new ValidationError(ValidationErrorCode.MarkdownParserError, - string.Concat(sourceFile.Name, ":", lineNumber), "Nested tags are not supported.")); - } + //if (inTag) + //{ + // // Nested tags not allowed + // LogMessage(new ValidationError(ValidationErrorCode.MarkdownParserError, + // string.Concat(sourceFile.Name, ":", lineNumber), "Nested tags are not supported.")); + //} string[] tags = GetTags(nextLine); LogMessage(new ValidationMessage(string.Concat(sourceFile.Name, ":", lineNumber), "Found TAGS line with {0}", string.Join(",", tags))); - inTag = true; - dropTagContent = !TagsAreIncluded(tags); - if (dropTagContent) + tagCount++; + + if (dropCount > 0 || !TagsAreIncluded(tags)) { - LogMessage(new ValidationMessage(string.Concat(sourceFile.Name, ":", lineNumber), "{0} are not found in the specified tags to include, content will be dropped.", string.Join(",", tags))); + dropCount++; + } + + if (dropCount == 1) + { + LogMessage(new ValidationMessage(string.Concat(sourceFile.Name, ":", lineNumber), + "{0} not found in the specified tags to include, content will be dropped.", string.Join(",", tags))); + } + else if (dropCount > 1) + { + LogMessage(new ValidationMessage(string.Concat(sourceFile.Name, ":", lineNumber), + "Dropping content due to containing [TAGS]")); } else { - // Replace line with div + // Keep line await writer.WriteLineAsync(nextLine); } continue; } + if (tagCount > 0 && dropCount > 0) + { + // Inside of a tag that shouldn't be included + LogMessage(new ValidationMessage(string.Concat(sourceFile.Name, ":", lineNumber), "Removing tagged content")); + continue; + } + + // Remove double blockquotes (">>") + if (IsDoubleBlockQuote(nextLine)) + { + LogMessage(new ValidationMessage(string.Concat(sourceFile.Name, ":", lineNumber), "Removing DoubleBlockQuote")); + continue; + } + + + await writer.WriteLineAsync(nextLine); } - if (inTag) + if (tagCount > 0) { // If inTag is true, there was a missing [END] tag somewhere LogMessage(new ValidationError(ValidationErrorCode.MarkdownParserError, From b56cd9461ec0c7c97507e08d003155827cebf9e3 Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Fri, 15 Jul 2016 11:48:21 -0400 Subject: [PATCH 04/17] Code cleanup Removing commented code I missed in last commit --- ApiDocs.Publishing/Tags/TagProcessor.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ApiDocs.Publishing/Tags/TagProcessor.cs b/ApiDocs.Publishing/Tags/TagProcessor.cs index 5153a4b..875977c 100644 --- a/ApiDocs.Publishing/Tags/TagProcessor.cs +++ b/ApiDocs.Publishing/Tags/TagProcessor.cs @@ -104,13 +104,6 @@ public async Task Preprocess(FileInfo sourceFile) // Check if this is a [TAGS] marker if (IsTagLine(nextLine, sourceFile.Name, lineNumber)) { - //if (inTag) - //{ - // // Nested tags not allowed - // LogMessage(new ValidationError(ValidationErrorCode.MarkdownParserError, - // string.Concat(sourceFile.Name, ":", lineNumber), "Nested tags are not supported.")); - //} - string[] tags = GetTags(nextLine); LogMessage(new ValidationMessage(string.Concat(sourceFile.Name, ":", lineNumber), "Found TAGS line with {0}", string.Join(",", tags))); From 97ab3e3210c7a09148c83ee3c13be834ec8833ef Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Tue, 26 Jul 2016 15:46:50 -0400 Subject: [PATCH 05/17] Moved include processing to preprocess So that includes can also be used in API testing --- ApiDocs.Publishing/Tags/TagProcessor.cs | 92 +++++++++++-------------- 1 file changed, 41 insertions(+), 51 deletions(-) diff --git a/ApiDocs.Publishing/Tags/TagProcessor.cs b/ApiDocs.Publishing/Tags/TagProcessor.cs index 875977c..e96a509 100644 --- a/ApiDocs.Publishing/Tags/TagProcessor.cs +++ b/ApiDocs.Publishing/Tags/TagProcessor.cs @@ -40,7 +40,7 @@ class TagProcessor private static Regex ValidTagFormat = new Regex(@"^\[TAGS=[-\.\w]+(?:,\s?[-\.\w]*)*\]", RegexOptions.IgnoreCase); private static Regex GetTagList = new Regex(@"\[TAGS=([-\.,\s\w]+)\]", RegexOptions.IgnoreCase); - private static Regex ConvertedIncludeFormat = new Regex(@"

\[INCLUDE\s*[-.\w]+\]

", RegexOptions.IgnoreCase); + private static Regex IncludeFormat = new Regex(@"\[INCLUDE\s*\[[-/.\w]+\]\(([-/.\w]+)\)\]", RegexOptions.IgnoreCase); private Action LogMessage = null; @@ -148,7 +148,35 @@ public async Task Preprocess(FileInfo sourceFile) continue; } - + // Import include file content + if (IsIncludeLine(nextLine)) + { + FileInfo includeFile = GetIncludeFile(nextLine, sourceFile); + if (!includeFile.Exists) + { + LogMessage(new ValidationError(ValidationErrorCode.ErrorOpeningFile, nextLine, "The included file {0} was not found", includeFile.FullName)); + continue; + } + + if (includeFile != null) + { + if (includeFile.FullName.Equals(sourceFile.FullName)) + { + LogMessage(new ValidationError(ValidationErrorCode.MarkdownParserError, nextLine, "A Markdown file cannot include itself")); + continue; + } + + string includeContent = await Preprocess(includeFile); + + await writer.WriteLineAsync(includeContent); + } + else + { + LogMessage(new ValidationError(ValidationErrorCode.ErrorReadingFile, nextLine, "Could not load include content from {0}", includeFile.FullName)); + } + + continue; + } await writer.WriteLineAsync(nextLine); } @@ -196,36 +224,6 @@ public async Task PostProcess(string html, FileInfo sourceFile, Markdown continue; } - // Load includes - if (IsConvertedIncludeLine(nextLine)) - { - FileInfo includeFile = GetIncludeFile(nextLine, sourceFile); - if (!includeFile.Exists) - { - LogMessage(new ValidationError(ValidationErrorCode.ErrorOpeningFile, nextLine, "The included file {0} was not found", includeFile.FullName)); - continue; - } - - if (includeFile != null) - { - if (includeFile.FullName.Equals(sourceFile.FullName)) - { - LogMessage(new ValidationError(ValidationErrorCode.MarkdownParserError, nextLine, "A Markdown file cannot include itself")); - continue; - } - - string includeContent = await GetIncludedContent(includeFile, converter); - - await writer.WriteLineAsync(includeContent); - } - else - { - LogMessage(new ValidationError(ValidationErrorCode.ErrorReadingFile, nextLine, "Could not load include content from {0}", includeFile.FullName)); - } - - continue; - } - await writer.WriteLineAsync(nextLine); } @@ -291,6 +289,16 @@ private bool IsEndLine(string text) return text.Trim().ToUpper().Equals("[END]"); } + private bool IsIncludeLine(string text) + { + if (text.ToUpper().Contains("INCLUDE")) + { + return IncludeFormat.IsMatch(text); + } + + return false; + } + private bool IsConvertedTagLine(string text) { // To handle edge case where you have a [TAGS] type entry inside @@ -308,19 +316,9 @@ private bool IsConvertedEndLine(string text) return text.Equals("

[END]

"); } - private bool IsConvertedIncludeLine(string text) - { - if (text.ToUpper().Contains("INCLUDE")) - { - return ConvertedIncludeFormat.IsMatch(text); - } - - return false; - } - private FileInfo GetIncludeFile(string text, FileInfo sourceFile) { - Match m = ConvertedIncludeFormat.Match(text); + Match m = IncludeFormat.Match(text); if (m.Success && m.Groups.Count == 2) { @@ -347,14 +345,6 @@ private string GetEndDivMarker() return ""; } - private async Task GetIncludedContent (FileInfo includeFile, Markdown converter) - { - // Do inline conversion, including pre and post processing - string html = converter.Transform(await Preprocess(includeFile)); - - return await PostProcess(html, includeFile, converter); - } - private void DefaultLogMessage(ValidationError msg) { // Empty method From b77209dfde52c960268218d2d56986d114575cc0 Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Tue, 26 Jul 2016 16:30:53 -0400 Subject: [PATCH 06/17] Moved TagProcessor to validation project Updated Docfile.Scan to preprocess content using TagProcessor. --- ApiDocs.Publishing/ApiDocs.Publishing.csproj | 1 - .../Html/DocumentPublisherHtml.cs | 4 ++-- ApiDocs.Validation/ApiDocs.Validation.csproj | 1 + ApiDocs.Validation/DocFile.cs | 9 +++++---- .../Tags/TagProcessor.cs | 18 +++++++++--------- 5 files changed, 17 insertions(+), 16 deletions(-) rename {ApiDocs.Publishing => ApiDocs.Validation}/Tags/TagProcessor.cs (95%) diff --git a/ApiDocs.Publishing/ApiDocs.Publishing.csproj b/ApiDocs.Publishing/ApiDocs.Publishing.csproj index be7d5e4..e13b6d8 100644 --- a/ApiDocs.Publishing/ApiDocs.Publishing.csproj +++ b/ApiDocs.Publishing/ApiDocs.Publishing.csproj @@ -61,7 +61,6 @@ - diff --git a/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs b/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs index cc4908e..4ea1485 100644 --- a/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs +++ b/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs @@ -36,7 +36,7 @@ namespace ApiDocs.Publishing.Html using ApiDocs.Validation.Writers; using MarkdownDeep; using Newtonsoft.Json; - using Tags; + using Validation.Tags; public class DocumentPublisherHtml : DocumentPublisher { @@ -234,7 +234,7 @@ protected override async Task PublishFileToDestinationAsync(FileInfo sourceFile, TagProcessor tagProcessor = new TagProcessor(PageParameters?["tags"]?.ToString(), LogMessage); var converter = this.GetMarkdownConverter(); - var html = converter.Transform(await tagProcessor.Preprocess(sourceFile)); + var html = converter.Transform(tagProcessor.Preprocess(sourceFile)); html = await tagProcessor.PostProcess(html, sourceFile, converter); var pageData = page.Annotation ?? new PageAnnotation(); diff --git a/ApiDocs.Validation/ApiDocs.Validation.csproj b/ApiDocs.Validation/ApiDocs.Validation.csproj index 53ece9e..c1723c5 100644 --- a/ApiDocs.Validation/ApiDocs.Validation.csproj +++ b/ApiDocs.Validation/ApiDocs.Validation.csproj @@ -88,6 +88,7 @@ + diff --git a/ApiDocs.Validation/DocFile.cs b/ApiDocs.Validation/DocFile.cs index bc4cace..d756932 100644 --- a/ApiDocs.Validation/DocFile.cs +++ b/ApiDocs.Validation/DocFile.cs @@ -32,6 +32,7 @@ namespace ApiDocs.Validation using System.Linq; using ApiDocs.Validation.Error; using ApiDocs.Validation.TableSpec; + using Tags; using MarkdownDeep; using Newtonsoft.Json; @@ -152,10 +153,10 @@ protected void TransformMarkdownIntoBlocksAndLinks(string inputMarkdown) protected virtual string GetContentsOfFile() { - using (StreamReader reader = File.OpenText(this.FullPath)) - { - return reader.ReadToEnd(); - } + // Preprocess file content + FileInfo docFile = new FileInfo(this.FullPath); + TagProcessor tagProcessor = new TagProcessor(string.Empty); + return tagProcessor.Preprocess(docFile); } diff --git a/ApiDocs.Publishing/Tags/TagProcessor.cs b/ApiDocs.Validation/Tags/TagProcessor.cs similarity index 95% rename from ApiDocs.Publishing/Tags/TagProcessor.cs rename to ApiDocs.Validation/Tags/TagProcessor.cs index e96a509..672f9da 100644 --- a/ApiDocs.Publishing/Tags/TagProcessor.cs +++ b/ApiDocs.Validation/Tags/TagProcessor.cs @@ -31,9 +31,9 @@ using System.Text.RegularExpressions; using MarkdownDeep; -namespace ApiDocs.Publishing.Tags +namespace ApiDocs.Validation.Tags { - class TagProcessor + public class TagProcessor { private string[] TagsToInclude = null; private static string[] tagSeparators = { ",", " " }; @@ -61,7 +61,7 @@ public TagProcessor(string tags, Action logMethod = null) ///
/// The file containing the Markdown contents to preprocess. /// The preprocessed contents of the file. - public async Task Preprocess(FileInfo sourceFile) + public string Preprocess(FileInfo sourceFile) { StringWriter writer = new StringWriter(); StreamReader reader = new StreamReader(sourceFile.OpenRead()); @@ -70,7 +70,7 @@ public async Task Preprocess(FileInfo sourceFile) int tagCount = 0; int dropCount = 0; string nextLine; - while ((nextLine = await reader.ReadLineAsync()) != null) + while ((nextLine = reader.ReadLine()) != null) { lineNumber++; @@ -82,7 +82,7 @@ public async Task Preprocess(FileInfo sourceFile) { if (dropCount <= 0) { - await writer.WriteLineAsync(nextLine); + writer.WriteLine(nextLine); } else { @@ -128,7 +128,7 @@ public async Task Preprocess(FileInfo sourceFile) else { // Keep line - await writer.WriteLineAsync(nextLine); + writer.WriteLine(nextLine); } continue; @@ -166,9 +166,9 @@ public async Task Preprocess(FileInfo sourceFile) continue; } - string includeContent = await Preprocess(includeFile); + string includeContent = Preprocess(includeFile); - await writer.WriteLineAsync(includeContent); + writer.WriteLine(includeContent); } else { @@ -178,7 +178,7 @@ public async Task Preprocess(FileInfo sourceFile) continue; } - await writer.WriteLineAsync(nextLine); + writer.WriteLine(nextLine); } if (tagCount > 0) From 64882977180a4e51d8e7e219f4697cef635e366c Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Tue, 26 Jul 2016 16:59:05 -0400 Subject: [PATCH 07/17] Moved parameters option to base options Allows parameters to be used across all scenarios, not just publish. --- ApiDocs.Console/CommandLineOptions.cs | 23 ++++++++++++++++--- ApiDocs.Console/Program.cs | 2 +- .../Html/DocumentPublisherHtml.cs | 17 +------------- ApiDocs.Validation/DocFile.cs | 8 +++---- ApiDocs.Validation/DocSet.cs | 4 ++-- ApiDocs.Validation/Writers/IPublishOptions.cs | 4 ++-- 6 files changed, 30 insertions(+), 28 deletions(-) diff --git a/ApiDocs.Console/CommandLineOptions.cs b/ApiDocs.Console/CommandLineOptions.cs index f3f2890..3caeb7a 100644 --- a/ApiDocs.Console/CommandLineOptions.cs +++ b/ApiDocs.Console/CommandLineOptions.cs @@ -95,6 +95,26 @@ class BaseOptions [Option("ignore-errors", HelpText="Prevent errors from generating a non-zero return code.")] public bool IgnoreErrors { get; set; } + [Option("parameters", HelpText = "Specify additional page variables that are used by the publishing engine. URL encoded: key=value&key2=value2.")] + public string AdditionalPageParameters { get; set; } + + public Dictionary PageParameterDict { + get + { + if (string.IsNullOrEmpty(AdditionalPageParameters)) + return null; + + var data = new Dictionary(); + + var parameters = Validation.Http.HttpParser.ParseQueryString(AdditionalPageParameters); + foreach (var key in parameters.AllKeys) + { + data[key] = parameters[key]; + } + return data; + } + } + #if DEBUG [Option("debug", HelpText="Launch the debugger before doing anything interesting")] public bool AttachDebugger { get; set; } @@ -378,9 +398,6 @@ public string[] FilesToPublish { set { this.SourceFiles = value.ComponentsJoinedByString(";"); } } - [Option("parameters", HelpText="Specify additional page variables that are used by the publishing engine. URL encoded: key=value&key2=value2.")] - public string AdditionalPageParameters { get; set; } - [Option("allow-unsafe-html", HelpText="Allows HTML tags in the markdown source to be passed through to the output markdown.")] public bool AllowUnsafeHtmlContentInMarkdown { get; set; } diff --git a/ApiDocs.Console/Program.cs b/ApiDocs.Console/Program.cs index 9a4463b..9444661 100644 --- a/ApiDocs.Console/Program.cs +++ b/ApiDocs.Console/Program.cs @@ -230,7 +230,7 @@ private static Task GetDocSetAsync(DocSetOptions options) FancyConsole.VerboseWriteLine("Scanning documentation files..."); ValidationError[] loadErrors; - if (!set.ScanDocumentation(out loadErrors)) + if (!set.ScanDocumentation(options.PageParameterDict?["tags"]?.ToString(), out loadErrors)) { FancyConsole.WriteLine("Errors detected while parsing documentation set:"); WriteMessages(loadErrors, false, " ", false); diff --git a/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs b/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs index 4ea1485..733430c 100644 --- a/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs +++ b/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs @@ -58,7 +58,7 @@ public DocumentPublisherHtml(DocSet docs, IPublishOptions options) TemplateHtmlFilename = options.TemplateFilename ?? "template.htm"; HtmlOutputExtension = options.OutputExtension ?? ".htm"; EnableHtmlTagPassThrough = options.AllowUnsafeHtmlContentInMarkdown; - PageParameters = GeneratePageParameters(options); + PageParameters = options.PageParameterDict; } /// @@ -334,21 +334,6 @@ private string ParseDocumentIfStatement(string key, string containingFilePath) } - private static Dictionary GeneratePageParameters(IPublishOptions options) - { - if (string.IsNullOrEmpty(options.AdditionalPageParameters)) - return null; - - var data = new Dictionary(); - - var parameters = Validation.Http.HttpParser.ParseQueryString(options.AdditionalPageParameters); - foreach (var key in parameters.AllKeys) - { - data[key] = parameters[key]; - } - return data; - } - // ReSharper disable once ClassNeverInstantiated.Local class IfQueryData { diff --git a/ApiDocs.Validation/DocFile.cs b/ApiDocs.Validation/DocFile.cs index d756932..2d12bef 100644 --- a/ApiDocs.Validation/DocFile.cs +++ b/ApiDocs.Validation/DocFile.cs @@ -151,11 +151,11 @@ protected void TransformMarkdownIntoBlocksAndLinks(string inputMarkdown) this.MarkdownLinks = new List(md.FoundLinks); } - protected virtual string GetContentsOfFile() + protected virtual string GetContentsOfFile(string tags) { // Preprocess file content FileInfo docFile = new FileInfo(this.FullPath); - TagProcessor tagProcessor = new TagProcessor(string.Empty); + TagProcessor tagProcessor = new TagProcessor(tags); return tagProcessor.Preprocess(docFile); } @@ -163,14 +163,14 @@ protected virtual string GetContentsOfFile() /// /// Read the contents of the file into blocks and generate any resource or method definitions from the contents /// - public bool Scan(out ValidationError[] errors) + public bool Scan(string tags, out ValidationError[] errors) { this.HasScanRun = true; List detectedErrors = new List(); try { - this.TransformMarkdownIntoBlocksAndLinks(this.GetContentsOfFile()); + this.TransformMarkdownIntoBlocksAndLinks(this.GetContentsOfFile(tags)); } catch (IOException ioex) { diff --git a/ApiDocs.Validation/DocSet.cs b/ApiDocs.Validation/DocSet.cs index ec16847..3589533 100644 --- a/ApiDocs.Validation/DocSet.cs +++ b/ApiDocs.Validation/DocSet.cs @@ -224,7 +224,7 @@ public static string ResolvePathWithUserRoot(string path) /// Scan all files in the documentation set to load /// information about resources and methods defined in those files /// - public bool ScanDocumentation(out ValidationError[] errors) + public bool ScanDocumentation(string tags, out ValidationError[] errors) { var foundResources = new List(); var foundMethods = new List(); @@ -245,7 +245,7 @@ public bool ScanDocumentation(out ValidationError[] errors) foreach (var file in this.Files) { ValidationError[] parseErrors; - if (!file.Scan(out parseErrors)) + if (!file.Scan(tags, out parseErrors)) { detectedErrors.AddRange(parseErrors); } diff --git a/ApiDocs.Validation/Writers/IPublishOptions.cs b/ApiDocs.Validation/Writers/IPublishOptions.cs index 4d80b6c..7916e0b 100644 --- a/ApiDocs.Validation/Writers/IPublishOptions.cs +++ b/ApiDocs.Validation/Writers/IPublishOptions.cs @@ -61,7 +61,7 @@ public interface IPublishOptions /// /// URL encoded string that contains additional parameters that can be used by the rendering engine /// - string AdditionalPageParameters { get; set; } + Dictionary PageParameterDict { get; } /// @@ -85,7 +85,7 @@ public class DefaultPublishOptions : IPublishOptions public string OutputExtension { get; set; } - public string AdditionalPageParameters { get; set; } + public Dictionary PageParameterDict { get; } public string TableOfContentsOutputRelativePath { get; set; } From 6a2118c1d3576fe66eedf039005143dee3381c5a Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Wed, 27 Jul 2016 09:13:57 -0400 Subject: [PATCH 08/17] Updated tests with new function signature --- ApiDocs.Validation.UnitTests/DocFileForTesting.cs | 2 +- .../ResourceStringValidationTests.cs | 2 +- ApiDocs.Validation.UnitTests/SchemaValidatorTests.cs | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ApiDocs.Validation.UnitTests/DocFileForTesting.cs b/ApiDocs.Validation.UnitTests/DocFileForTesting.cs index 0cc1996..8761d08 100644 --- a/ApiDocs.Validation.UnitTests/DocFileForTesting.cs +++ b/ApiDocs.Validation.UnitTests/DocFileForTesting.cs @@ -37,7 +37,7 @@ public DocFileForTesting(string contentsOfFile, string fullPath, string displayN this.Parent = parent; } - protected override string GetContentsOfFile() + protected override string GetContentsOfFile(string tags) { return this.contentsOfFile; } diff --git a/ApiDocs.Validation.UnitTests/ResourceStringValidationTests.cs b/ApiDocs.Validation.UnitTests/ResourceStringValidationTests.cs index fb83117..3cc666a 100644 --- a/ApiDocs.Validation.UnitTests/ResourceStringValidationTests.cs +++ b/ApiDocs.Validation.UnitTests/ResourceStringValidationTests.cs @@ -41,7 +41,7 @@ static DocFile GetDocFile() DocFile testFile = new DocFileForTesting(Resources.ExampleResources, "\resources.md", "\resources.md", docSet); ValidationError[] detectedErrors; - testFile.Scan(out detectedErrors); + testFile.Scan(string.Empty, out detectedErrors); Assert.IsFalse(detectedErrors.WereWarningsOrErrors(), "Detected warnings or errors when reading the example markdown file."); diff --git a/ApiDocs.Validation.UnitTests/SchemaValidatorTests.cs b/ApiDocs.Validation.UnitTests/SchemaValidatorTests.cs index 4b133da..3f8d61b 100644 --- a/ApiDocs.Validation.UnitTests/SchemaValidatorTests.cs +++ b/ApiDocs.Validation.UnitTests/SchemaValidatorTests.cs @@ -159,7 +159,7 @@ public void TruncatedExampleWithRequiredPropertiesTest() DocFile testFile = new DocFileForTesting(Resources.ExampleValidateResponse, "\test\test.md", "test.md", docSet); ValidationError[] detectedErrors; - testFile.Scan(out detectedErrors); + testFile.Scan(string.Empty, out detectedErrors); Assert.IsEmpty(detectedErrors.Where(x => x.IsError)); @@ -185,7 +185,7 @@ public void TruncatedExampleSelectStatementOnChildren() DocFile testFile = new DocFileForTesting(Resources.ExampleValidationSelectStatement, "\test\test.md", "test.md", docSet); ValidationError[] detectedErrors; - testFile.Scan(out detectedErrors); + testFile.Scan(string.Empty, out detectedErrors); Assert.IsEmpty(detectedErrors.Where(x => x.IsError)); @@ -209,7 +209,7 @@ public void TruncatedExampleSelectStatementOnChildrenExpectFailure() DocFile testFile = new DocFileForTesting(Resources.ExampleValidationSelectStatementFailure, "\test\test.md", "test.md", docSet); ValidationError[] detectedErrors; - testFile.Scan(out detectedErrors); + testFile.Scan(string.Empty, out detectedErrors); Assert.IsEmpty(detectedErrors.Where(x => x.IsError)); From 260187cff05e95de761c9b0722f23cb9d4d49b35 Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Thu, 28 Jul 2016 12:30:26 -0400 Subject: [PATCH 09/17] Added support for rooted include paths You can now set a path in an INCLUDE tag that is relative to the root of the doc set. --- .../Html/DocumentPublisherHtml.cs | 3 ++- ApiDocs.Validation/DocFile.cs | 2 +- ApiDocs.Validation/Tags/TagProcessor.cs | 20 +++++++++++++++++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs b/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs index 733430c..838c5d9 100644 --- a/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs +++ b/ApiDocs.Publishing/Html/DocumentPublisherHtml.cs @@ -231,7 +231,8 @@ protected override async Task PublishFileToDestinationAsync(FileInfo sourceFile, var destinationPath = this.GetPublishedFilePath(sourceFile, destinationRoot, HtmlOutputExtension); // Create a tag processor - TagProcessor tagProcessor = new TagProcessor(PageParameters?["tags"]?.ToString(), LogMessage); + TagProcessor tagProcessor = new TagProcessor(PageParameters?["tags"]?.ToString(), + page.Parent.SourceFolderPath, LogMessage); var converter = this.GetMarkdownConverter(); var html = converter.Transform(tagProcessor.Preprocess(sourceFile)); diff --git a/ApiDocs.Validation/DocFile.cs b/ApiDocs.Validation/DocFile.cs index 2d12bef..9b04865 100644 --- a/ApiDocs.Validation/DocFile.cs +++ b/ApiDocs.Validation/DocFile.cs @@ -155,7 +155,7 @@ protected virtual string GetContentsOfFile(string tags) { // Preprocess file content FileInfo docFile = new FileInfo(this.FullPath); - TagProcessor tagProcessor = new TagProcessor(tags); + TagProcessor tagProcessor = new TagProcessor(tags, Parent.SourceFolderPath); return tagProcessor.Preprocess(docFile); } diff --git a/ApiDocs.Validation/Tags/TagProcessor.cs b/ApiDocs.Validation/Tags/TagProcessor.cs index 672f9da..97eec1c 100644 --- a/ApiDocs.Validation/Tags/TagProcessor.cs +++ b/ApiDocs.Validation/Tags/TagProcessor.cs @@ -36,6 +36,7 @@ namespace ApiDocs.Validation.Tags public class TagProcessor { private string[] TagsToInclude = null; + private string DocSetRoot = null; private static string[] tagSeparators = { ",", " " }; private static Regex ValidTagFormat = new Regex(@"^\[TAGS=[-\.\w]+(?:,\s?[-\.\w]*)*\]", RegexOptions.IgnoreCase); @@ -44,7 +45,7 @@ public class TagProcessor private Action LogMessage = null; - public TagProcessor(string tags, Action logMethod = null) + public TagProcessor(string tags, string docSetRoot, Action logMethod = null) { if (!string.IsNullOrEmpty(tags)) { @@ -52,6 +53,8 @@ public TagProcessor(string tags, Action logMethod = null) StringSplitOptions.RemoveEmptyEntries); } + DocSetRoot = docSetRoot; + // If not logging method supplied default to a no-op LogMessage = logMethod ?? DefaultLogMessage; } @@ -324,7 +327,20 @@ private FileInfo GetIncludeFile(string text, FileInfo sourceFile) { string relativePath = Path.ChangeExtension(m.Groups[1].Value, "md"); - return new FileInfo(Path.Combine(sourceFile.Directory.FullName, relativePath)); + string fullPathToIncludeFile = string.Empty; + + if (Path.IsPathRooted(relativePath)) + { + // Path is relative to the root of the doc set + relativePath = relativePath.TrimStart('/'); + fullPathToIncludeFile = Path.Combine(DocSetRoot, relativePath); + } + else + { + fullPathToIncludeFile = Path.Combine(sourceFile.Directory.FullName, relativePath); + } + + return new FileInfo(fullPathToIncludeFile); } return null; From 2df987f0132001c9bb1ac2beeb67e2a37c4d81bd Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Fri, 29 Jul 2016 14:14:58 -0400 Subject: [PATCH 10/17] Fix case mismatch bugs Couple of bugs exposed by mismatching tag case. - Using uppercase "TAGS=" in the parameters command line param would not cause the options parser to match, causing an uncaught exception. - Using lowercase tag values in markdown would cause it not to match. --- ApiDocs.Console/CommandLineOptions.cs | 2 +- ApiDocs.Console/Program.cs | 10 +++++++++- ApiDocs.Validation/Tags/TagProcessor.cs | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ApiDocs.Console/CommandLineOptions.cs b/ApiDocs.Console/CommandLineOptions.cs index 3caeb7a..a08e884 100644 --- a/ApiDocs.Console/CommandLineOptions.cs +++ b/ApiDocs.Console/CommandLineOptions.cs @@ -106,7 +106,7 @@ public Dictionary PageParameterDict { var data = new Dictionary(); - var parameters = Validation.Http.HttpParser.ParseQueryString(AdditionalPageParameters); + var parameters = Validation.Http.HttpParser.ParseQueryString(AdditionalPageParameters.ToLower()); foreach (var key in parameters.AllKeys) { data[key] = parameters[key]; diff --git a/ApiDocs.Console/Program.cs b/ApiDocs.Console/Program.cs index 9444661..7b34e06 100644 --- a/ApiDocs.Console/Program.cs +++ b/ApiDocs.Console/Program.cs @@ -230,7 +230,15 @@ private static Task GetDocSetAsync(DocSetOptions options) FancyConsole.VerboseWriteLine("Scanning documentation files..."); ValidationError[] loadErrors; - if (!set.ScanDocumentation(options.PageParameterDict?["tags"]?.ToString(), out loadErrors)) + + string tagsToInclude = string.Empty; + if (options.PageParameterDict != null && + options.PageParameterDict.ContainsKey("tags")) + { + tagsToInclude = options.PageParameterDict["tags"]?.ToString(); + } + + if (!set.ScanDocumentation(tagsToInclude, out loadErrors)) { FancyConsole.WriteLine("Errors detected while parsing documentation set:"); WriteMessages(loadErrors, false, " ", false); diff --git a/ApiDocs.Validation/Tags/TagProcessor.cs b/ApiDocs.Validation/Tags/TagProcessor.cs index 97eec1c..1cc259d 100644 --- a/ApiDocs.Validation/Tags/TagProcessor.cs +++ b/ApiDocs.Validation/Tags/TagProcessor.cs @@ -261,7 +261,7 @@ private string[] GetTags(string text) if (m.Success && m.Groups.Count == 2) { - return m.Groups[1].Value.Split(TagProcessor.tagSeparators, + return m.Groups[1].Value.ToUpper().Split(TagProcessor.tagSeparators, StringSplitOptions.RemoveEmptyEntries); } From b4d3fbe7c373a3f9892ac1b724975b80c301b055 Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Tue, 2 Aug 2016 11:14:59 -0400 Subject: [PATCH 11/17] Added blank lines after TAG and before END To allow authors to not insert blank lines themselves. Otherwise, MarkdownDeep would get the

tags wrong, resulting in funky HTML markup. --- ApiDocs.Validation/Tags/TagProcessor.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ApiDocs.Validation/Tags/TagProcessor.cs b/ApiDocs.Validation/Tags/TagProcessor.cs index 1cc259d..74efc5b 100644 --- a/ApiDocs.Validation/Tags/TagProcessor.cs +++ b/ApiDocs.Validation/Tags/TagProcessor.cs @@ -85,6 +85,8 @@ public string Preprocess(FileInfo sourceFile) { if (dropCount <= 0) { + // To keep output clean if author did not insert blank line before + writer.WriteLine(""); writer.WriteLine(nextLine); } else @@ -132,6 +134,8 @@ public string Preprocess(FileInfo sourceFile) { // Keep line writer.WriteLine(nextLine); + // To keep output clean if author did not insert blank line after + writer.WriteLine(""); } continue; From 3559fd7fd7398488dfc3c3c0d5a053241e5e548a Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Tue, 2 Aug 2016 11:22:11 -0400 Subject: [PATCH 12/17] Corrected action for check-links --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index dfdf1c0..610ef38 100644 --- a/readme.md +++ b/readme.md @@ -63,7 +63,7 @@ Check for broken links in the documentation. No specific options are required. Using `--verbose` will include warnings about links that were not verified. -Example: `apidocs.exe links --path ~/github/api-docs --method search` +Example: `apidocs.exe check-links --path ~/github/api-docs --method search` ### Check-docs Command The `check-docs` command ensures that the documentation is internally consistent. From 0c7695afaa39e72e6e23128fc136e5c27932553b Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Tue, 2 Aug 2016 11:51:44 -0400 Subject: [PATCH 13/17] Removed errors in check-links for TAGS and END These were being validated as if they were links by ID, and check-links was reporting them as errors. --- ApiDocs.Validation/DocFile.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ApiDocs.Validation/DocFile.cs b/ApiDocs.Validation/DocFile.cs index 9b04865..f0513c3 100644 --- a/ApiDocs.Validation/DocFile.cs +++ b/ApiDocs.Validation/DocFile.cs @@ -929,7 +929,13 @@ public bool ValidateNoBrokenLinks(bool includeWarnings, out ValidationError[] er { if (null == link.Definition) { - foundErrors.Add(new ValidationError(ValidationErrorCode.MissingLinkSourceId, this.DisplayName, "Link specifies ID '{0}' which was not found in the document.", link.Text)); + // Don't treat TAGS or END markers like links + if (!link.Text.ToUpper().Equals("END") && !link.Text.ToUpper().StartsWith("TAGS=")) + { + foundErrors.Add(new ValidationError(ValidationErrorCode.MissingLinkSourceId, this.DisplayName, + "Link specifies ID '{0}' which was not found in the document.", link.Text)); + } + continue; } From 218c44c5df8cc8036cd22b26168a130c79affa56 Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Wed, 3 Aug 2016 11:25:04 -0400 Subject: [PATCH 14/17] Added docs for new features --- docs/markdown-customizations.md | 117 ++++++++++++++++++++++++++++++++ docs/markdown-requirements.md | 4 ++ readme.md | 1 + 3 files changed, 122 insertions(+) create mode 100644 docs/markdown-customizations.md diff --git a/docs/markdown-customizations.md b/docs/markdown-customizations.md new file mode 100644 index 0000000..44f1916 --- /dev/null +++ b/docs/markdown-customizations.md @@ -0,0 +1,117 @@ +# Markdown customizations + +Markdown-scanner implements the following additional features on top of the Markdown language. + +- [Tagging of content](#tagging-of-content) +- [Including other Markdown files](#including-other-markdown-files) + +## Tagging of content + +Tagging of content allows the author to include sections that are conditionally included or omitted based on the parameters passed at build time. This allows one set of source files to generate content for multiple targets. + +### Syntax + +To tag content, wrap the Markdown to tag with the following tags: + +```Markdown +[TAGS=] + +Some Markdown content here + +[END] +``` + +This will cause everything between the start `[TAGS]` line and the ending `[END]` line to only be included in the final output if one or more of the tags in the comma-delimited list are specified at build time. + +Included content will be wrapped with `

` tags with a `class` attribute set based on the tags. The class value takes the form `content-`. So a Markdown marker of `[TAGS=FOO]` would result in `
`. + +#### Limitations + +There are some limits to what you can do with this. + +- Both the `[TAGS]` marker and the `[END]` marker must be on their own line. You cannot insert them in the middle of a paragraph. + +### Specifying tags at build time + +To specify tags at build time, the `--parameters` parameter must include a `TAGS` key, with the value set to a comma-delimited list of tags to include. For example: + +```Shell +apidocs.exe publish --format html --path .\src --output .\out --parameters "TAGS=OUTLOOK,v2" +``` + +### Example + +Let's take a look at a simple example. Suppose you have the following Markdown source file: + +```Markdown +# Tagging Demo + +This content should always appear in the output because it is not tagged. + +[TAGS=V1] +This is v1 content, and only appears if the V1 tag is specified at build. +[END] + +[TAGS=V2] +This is v2 content, and only appears if the V2 tag is specified at build. +[END] + +[TAGS=V1,V2] +This is v1 and v2 content, and only appears if either the V1 or V2 tags are specified at build. Also appears if both tags are specified. +[END] +``` + +If the `--parameters` parameter is omitted, or does not contain a `TAGS` key, or the `TAGS` key does not contain a `V1` or `V2` value, the following is the result: + +```html +

Tagging Demo

+

This content should always appear in the output because it is not tagged.

+``` + +If the `--parameters` parameter is set to `TAGS=V1`, the following is the result: + +```html +

Tagging Demo

+

This content should always appear in the output because it is not tagged.

+
+

This is v1 content, and only appears if the V1 tag is specified at build.

+
+
+

This is v1 and v2 content, and only appears if either the V1 or V2 tags are specified at build. Also appears if both tags are specified.

+
+``` + +## Including other Markdown files + +Use the following syntax to include another Markdown file. + +```Markdown +[INCLUDE [](path to file)] +``` + +For example: + +```Markdown +[INCLUDE [included-file.md](includes/included-file.md)] +``` + +### Paths to include files + +You can construct paths to include files as either relative to the root of the documentation set, or relative to the current file. + +#### Relative to root + +Paths that start with a `/` are considered relative to the root of the documentation set. + +```Markdown +[INCLUDE [included-file.md](/includes/included-file.md)] +``` + +#### Relative to the current file + +Paths that start with one or more `.`, or alpha-numeric characters, are considered relative to the current file. + +```Markdown +[INCLUDE [included-file.md](./includes/included-file.md)] +[INCLUDE [included-file.md](includes/included-file.md)] +``` \ No newline at end of file diff --git a/docs/markdown-requirements.md b/docs/markdown-requirements.md index 8dc128e..21e4c28 100644 --- a/docs/markdown-requirements.md +++ b/docs/markdown-requirements.md @@ -19,6 +19,10 @@ To work with this tool, the source documentation has a few basic requirements: Markdown supports the GitHub flavored markdown format. This is less useful for the automated test scenarios, but is required for HTML publishing. +## Markdown customizations + +Markdown scanner provides additional features on top of the Markdown language. For details, see [Markdown customizations](/docs/markdown-customizations.md). + ## Code blocks Markdown scanner recognizes fenced code blocks with three back-tick characters. diff --git a/readme.md b/readme.md index 610ef38..57fe9c8 100644 --- a/readme.md +++ b/readme.md @@ -44,6 +44,7 @@ All commands have the following options available: | `--short` | Print concise output to the console. | | `--verbose` | Print verbose output to the console, including full HTTP requests/responses. | | `--log ` | Log console output to a file. | +| `--parameters ` | A URL-encoded string containing key/value pairs. Allows additional parameters to be passed to the task. Currently used by the tagging feature to specify content to include. For more information see [Markdown customizations](docs/markdown-customizations.md). | ### Print Command Print information about the source files, resources, methods, and requests From 653822d55bcb08697589a63e3663583f35719539 Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Fri, 5 Aug 2016 14:59:44 -0400 Subject: [PATCH 15/17] Fixed issue with relative path If you set the --path parameter to a relative path like '.\source', the app fails to process. This is because it checked the source file's absolute path against the relative path and decided they weren't the same. Fixed this to set the DocSet root path to the absolute path. --- ApiDocs.Validation/DocSet.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ApiDocs.Validation/DocSet.cs b/ApiDocs.Validation/DocSet.cs index 3589533..abd1edb 100644 --- a/ApiDocs.Validation/DocSet.cs +++ b/ApiDocs.Validation/DocSet.cs @@ -217,7 +217,8 @@ public static string ResolvePathWithUserRoot(string path) var userFolderPath = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); return Path.Combine(userFolderPath, path.Substring(2)); } - return path; + // Return absolute path + return Path.GetFullPath(path); } /// From 91b6d17172666ad5356749cea690f83d3b8a89ed Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Thu, 11 Aug 2016 10:25:53 -0400 Subject: [PATCH 16/17] Fix for AppVeyor error AppVeyor reported: Writers\IPublishOptions.cs(88,62): error CS0840: 'ApiDocs.Validation.Writers.DefaultPublishOptions.PageParameterDict.get' must declare a body because it is not marked abstract or extern. Automatically implemented properties must define both get and set accessors. Added set accessor. --- ApiDocs.Validation/Writers/IPublishOptions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ApiDocs.Validation/Writers/IPublishOptions.cs b/ApiDocs.Validation/Writers/IPublishOptions.cs index 7916e0b..ac00d0b 100644 --- a/ApiDocs.Validation/Writers/IPublishOptions.cs +++ b/ApiDocs.Validation/Writers/IPublishOptions.cs @@ -85,7 +85,7 @@ public class DefaultPublishOptions : IPublishOptions public string OutputExtension { get; set; } - public Dictionary PageParameterDict { get; } + public Dictionary PageParameterDict { get; set; } public string TableOfContentsOutputRelativePath { get; set; } From 8a9294aa13cf3a2ab041143e12f5ba46a290b390 Mon Sep 17 00:00:00 2001 From: Jason Johnston Date: Fri, 12 Aug 2016 09:41:58 -0400 Subject: [PATCH 17/17] Added comment to trigger appveyor --- ApiDocs.Validation/Writers/IPublishOptions.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/ApiDocs.Validation/Writers/IPublishOptions.cs b/ApiDocs.Validation/Writers/IPublishOptions.cs index ac00d0b..d2561c4 100644 --- a/ApiDocs.Validation/Writers/IPublishOptions.cs +++ b/ApiDocs.Validation/Writers/IPublishOptions.cs @@ -85,6 +85,7 @@ public class DefaultPublishOptions : IPublishOptions public string OutputExtension { get; set; } + // Added set to make AppVeyor happy public Dictionary PageParameterDict { get; set; } public string TableOfContentsOutputRelativePath { get; set; }