Enhancement (configuration): Add keepescape mode 4 to repair Pandoc escapes

ConvertOneNote2MarkDown-v2.Tests.ps1

@@ -407,6 +407,93 @@ Describe "Encode-Markdown" -Tag 'Unit' {
 
 }
 
+Describe "Repair-MarkdownEscapes" -Tag 'Unit' {
+
+    Context 'Behavior' {
+
+        It "Should turn escaped list markers into lists (<Name>)" -TestCases @(
+            @{ Name = 'bullet'; Content = '\- one'; Expected = '- one' }
+            @{ Name = 'indented bullet'; Content = '  \- one'; Expected = '  - one' }
+            @{ Name = 'number'; Content = '7\. one'; Expected = '7. one' }
+            @{ Name = 'number alone before its text'; Content = "7\.`n`nitem text"; Expected = '7. item text' }
+        ) {
+            param ($Name, $Content, $Expected)
+
+            Repair-MarkdownEscapes -Content $Content | Should -BeExactly $Expected
+        }
+
+        It "Should accept a non-breaking space after an escaped list marker" {
+            $nbsp = [char]0x00A0
+            Repair-MarkdownEscapes -Content "\-$nbsp one`n7\.$nbsp two" | Should -BeExactly "- one`n7. two"
+        }
+
+        It "Should repair lists inside block quotes (keepspaces removes the '>' later)" {
+            Repair-MarkdownEscapes -Content "> 1\. one`n>`n> 2\. two`n>`n> \- x" | Should -BeExactly "> 1. one`n> 2. two`n> - x"
+        }
+
+        It "Should drop a lone dash line that only had a line-end backslash" {
+            Repair-MarkdownEscapes -Content "- one`n-\`n`n`n" | Should -BeExactly "- one`n`n`n"
+        }
+
+        It "Should remove blank lines between list items" {
+            Repair-MarkdownEscapes -Content "\- a`n`n\- b`n`n\- c" | Should -BeExactly "- a`n- b`n- c"
+            Repair-MarkdownEscapes -Content "1\. a`n`n2\. b" | Should -BeExactly "1. a`n2. b"
+        }
+
+        It "Should remove several blank lines, also with non-breaking spaces, between list items" {
+            Repair-MarkdownEscapes -Content "- a`n`n`n$( [char]0x00A0 )`n`n- b" | Should -BeExactly "- a`n- b"
+            Repair-MarkdownEscapes -Content "1. a`n`n`n`n2. b`n`ntext" | Should -BeExactly "1. a`n2. b`n`ntext"
+            Repair-MarkdownEscapes -Content "- a`n`n> $( [char]0x00A0 ) `n`n- b" | Should -BeExactly "- a`n- b"
+        }
+
+        It "Should keep blank lines between a list and a paragraph" {
+            Repair-MarkdownEscapes -Content "- a`n`ntext" | Should -BeExactly "- a`n`ntext"
+        }
+
+        It "Should remove lone dash lines, also with non-breaking spaces" {
+            Repair-MarkdownEscapes -Content "text`n-`n\-`n-$( [char]0x00A0 )`nmore" | Should -BeExactly "text`nmore"
+        }
+
+        It "Should turn a backslash at the end of a line into two spaces, and drop it at the end of a paragraph" {
+            Repair-MarkdownEscapes -Content "one\`ntwo\`n`nnext" | Should -BeExactly "one  `ntwo`n`nnext"
+        }
+
+        It "Should remove escapes that can't change the rendering" {
+            Repair-MarkdownEscapes -Content 'a --\> b, snake\_case, 1\. no, end\.' | Should -BeExactly 'a --> b, snake_case, 1. no, end.'
+        }
+
+        It "Should keep escapes that change the rendering (<Name>)" -TestCases @(
+            @{ Name = 'quote at line start'; Content = '\> not a quote' }
+            @{ Name = 'underscore at line start'; Content = '\_not emphasis_' }
+            @{ Name = 'image-like'; Content = 'a \![b](c)' }
+            @{ Name = 'pair of asterisks'; Content = 'a \*b\* c' }
+            @{ Name = 'brackets'; Content = '\[x\]' }
+        ) {
+            param ($Name, $Content)
+
+            Repair-MarkdownEscapes -Content $Content | Should -BeExactly $Content
+        }
+
+        It "Should keep Windows and UNC paths" {
+            $content = 'C:\Program Files\App and \\server\share\dir'
+            Repair-MarkdownEscapes -Content $content | Should -BeExactly $content
+        }
+
+        It "Should not touch code" {
+            $content = "``\- keep \_this`` and`n`n``````text`n\- keep`n7\. keep\`n``````"
+            Repair-MarkdownEscapes -Content $content | Should -BeExactly $content
+        }
+
+        It "Should give the same result when run twice" {
+            $content = "\- a`n`n\- b\`nc`n`n7\. x --\> y\_z\`n`nend\"
+            $once = Repair-MarkdownEscapes -Content $content
+            Repair-MarkdownEscapes -Content $once | Should -BeExactly $once
+        }
+
+    }
+
+}
+
 Describe "New-OneNoteConnection" -Tag 'Unit' {
 
     Context 'Behavior' {
@@ -1380,6 +1467,27 @@ foo\bar\0
             }
         }
 
+        It "Should leave backslashes to Repair-MarkdownEscapes if config keepescape is 4" {
+            $params['Config']['keepescape']['value'] = 4
+
+            $result = @( New-SectionGroupConversionConfig @params 6>$null )
+            $result.Count | Should -Be 48
+
+            foreach ($pageCfg in $result) {
+                $fakeMarkdownContent = 'hello\$\^\\\*\_\[\]world'
+
+                # Mutate
+                foreach ($m in $pageCfg['mutations']) {
+                    foreach ($r in $m['replacements']) {
+                        $fakeMarkdownContent = $fakeMarkdownContent -replace $r['searchRegex'], $r['replacement']
+                    }
+                }
+
+                # None of the mutations may touch backslashes
+                $fakeMarkdownContent | Should -Match ([regex]::Escape('hello\$\^\\\*\_\[\]world'))
+            }
+        }
+
         It "Should honor config newlineCharacter" {
             $params['Config']['newlineCharacter']['value'] = 1
 

ConvertOneNote2MarkDown-v2.ps1

@@ -166,10 +166,11 @@ Whether to clear escape symbols from md files. See: https://pandoc.org/MANUAL.ht
 1: Clear all '\' characters  - Default
 2: Clear all '\' characters except those preceding alphanumeric characters
 3: Keep '\' symbol escape
+4: Keep '\' only where it changes the rendering: rebuild lists and line breaks that pandoc escaped ('\- item', '7\. item', '\' at the end of a line), remove other escapes that don't matter
 '@
             default = 1
             value = 1
-            validateRange = 1,3
+            validateRange = 1,4
         }
         newlineCharacter = @{
             description = @'
@@ -497,6 +498,107 @@ Function Encode-Markdown {
     $Name
 }
 
+Function Repair-MarkdownEscapes {
+    [CmdletBinding()]
+    param (
+        # The markdown content produced by pandoc (any newline style)
+        [Parameter(Mandatory,Position = 0,ValueFromPipeline = $true)]
+        [AllowEmptyString()]
+        [string]
+        $Content
+    )
+
+    process {
+        # Pandoc escapes text that merely looks like markdown. OneNote pages often start paragraphs with a literal '- ' or '7.' (no real list),
+        # which ends up as '\- item' or '7\. item'. Rebuild those lists, and remove only the escapes that can't change how the text is rendered.
+        # Fenced code blocks and inline code are never touched.
+        $saved = New-Object 'System.Collections.Generic.List[string]'
+        $shield = [System.Text.RegularExpressions.MatchEvaluator]{
+            param ($m)
+            $saved.Add($m.Value)
+            "MDSHIELD$( $saved.Count - 1 )END"
+        }
+        $text = $Content -replace '\r\n', "`n"
+        $text = [regex]::Replace($text, '(?ms)(?:^[ \t]*)?(`{3,})[^\n`]*\n.*?^[ \t]*\1`*[ \t]*$', $shield)
+        $text = [regex]::Replace($text, '(`+)(?!`)(.+?)(?<!`)\1(?!`)', $shield)
+
+        # A backslash at the end of a line is a hard break only in some renderers: use two spaces. At the end of a paragraph it has no effect.
+        $text = $text -replace '(?m)(?<!\\)\\$(?=\n[^\n])', '  '
+        $text = $text -replace '(?m)(?<!\\)\\[ \t]*$', ''
+        $text = $text -replace '(?m) +$(?=\n[ \t]*\n)', ''
+
+        # Lists
+        $text = $text -replace '(?m)^[ \t\u00A0>]*\\?-[ \t\u00A0]*$\n?', '' # A lone '-' or '\-' line is an empty bullet (below a text line, '-' would even make a heading)
+        $text = $text -replace '(?m)^((?:[ \t]*>)*[ \t]*)\\-[ \t\u00A0]+', '$1- '
+        $text = $text -replace '(?m)^(\d+)\\\.[ \t]*\n(?:[ \t]*\n)+(?=\S)', '$1. ' # A number alone on its line, with the item text in the next paragraph
+        $text = $text -replace '(?m)^((?:[ \t]*>)*[ \t]*\d+)\\\.[ \t\u00A0]+', '$1. '
+        $listStart = '^[ \t\u00A0>]*(?:[-*+]|\d+[.)])\s'
+        $lines = $text -split "`n"
+        $isBlank = '^[ \t\u00A0>]*$' # Lines with only non-breaking spaces or an empty block quote ('> ') become blank later on (keepspaces)
+        $text = @(
+            for ($i = 0; $i -lt $lines.Count; $i++) {
+                # Drop the blank lines between two list items
+                if ($lines[$i] -match $isBlank -and $i -gt 0) {
+                    $prev = $i - 1
+                    while ($prev -gt 0 -and $lines[$prev] -match $isBlank) { $prev-- }
+                    $next = $i + 1
+                    while ($next -lt $lines.Count - 1 -and $lines[$next] -match $isBlank) { $next++ }
+                    if ($lines[$prev] -match $listStart -and $next -lt $lines.Count -and $lines[$next] -match $listStart) {
+                        continue
+                    }
+                }
+                $lines[$i]
+            }
+        ) -join "`n"
+
+        # Escapes that can't change the rendering
+        $unescape = [System.Text.RegularExpressions.MatchEvaluator]{
+            param ($m)
+            if ($m.Value -eq '\\') { return $m.Value } # An escaped backslash is matched as a pair, so it can't hide the character after it
+            $c = $m.Value[1]
+            $start = $m.Index
+            $lineStart = if ($start -gt 0) { $text.LastIndexOf([char]10, $start - 1) + 1 } else { 0 }
+            $prefix = $text.Substring($lineStart, $start - $lineStart)
+            $before = if ($start -gt 0) { [string]$text[$start - 1] } else { '' }
+            $afterIndex = $start + 2
+            $after = if ($afterIndex -lt $text.Length) { [string]$text[$afterIndex] } else { '' }
+            $atLineStart = $prefix.Trim(" `t>").Length -eq 0
+            if (',."''%@/?!=+-#>'.Contains($c)) {
+                $keep = ('=+-#>'.Contains($c) -and $atLineStart) -or ($c -eq '.' -and $prefix -match '^[ \t>]*\d+$') -or ($c -eq '!' -and $after -eq '[')
+                if (!$keep -and $c -eq '#') {
+                    $lineEnd = $text.IndexOf([char]10, $start)
+                    $line = if ($lineEnd -lt 0) { $text.Substring($lineStart) } else { $text.Substring($lineStart, $lineEnd - $lineStart) }
+                    $keep = $line -match '^\s*#{1,6}(?:\s|$)' # A closing '\#' in a heading stays
+                }
+            }elseif ($c -eq '*' -or $c -eq '_') {
+                $loneGap = ($before -eq '' -or $before -match '^\s$') -and ($after -eq '' -or $after -match '^\s$')
+                $intraword = $before -match '^[\p{L}\p{N}]$' -and $after -match '^[\p{L}\p{N}]$'
+                if ($c -eq '*') {
+                    $pStart = if ($start -ge 2) { $text.LastIndexOf("`n`n", $start - 1, [System.StringComparison]::Ordinal) } else { -1 }
+                    $pEnd = $text.IndexOf("`n`n", $start, [System.StringComparison]::Ordinal)
+                    $from = if ($pStart -ge 0) { $pStart + 2 } else { 0 }
+                    $to = if ($pEnd -ge 0) { $pEnd } else { $text.Length }
+                    $paragraph = $text.Substring($from, $to - $from)
+                    $keep = $atLineStart -or !($loneGap -or ($paragraph.Length - $paragraph.Replace('*', '').Length) -eq 1)
+                }else {
+                    $keep = $atLineStart -or !($loneGap -or $intraword)
+                }
+            }else {
+                $keep = $true
+            }
+            if ($keep) { $m.Value } else { [string]$c }
+        }
+        $text = [regex]::Replace($text, '\\\\|\\.', $unescape)
+
+        # Restore code (fences can contain inline code and vice versa, so repeat until nothing is left)
+        $restore = [System.Text.RegularExpressions.MatchEvaluator]{ param ($m) $saved[[int]$m.Groups[1].Value] }
+        while ($text -match 'MDSHIELD\d+END') {
+            $text = [regex]::Replace($text, 'MDSHIELD(\d+)END', $restore)
+        }
+        $text
+    }
+}
+
 Function Set-ContentNoBom {
     [CmdletBinding()]
     param (
@@ -1282,6 +1384,9 @@ Function Convert-OneNotePage {
                             ''
                         }
                     ) -join "`n"
+                    if ($config['keepescape']['value'] -eq 4) {
+                        $content = Repair-MarkdownEscapes -Content $content
+                    }
                 }
 
                 # Mutate

README.md

@@ -30,7 +30,7 @@ The powershell script `ConvertOneNote2MarkDown-v2.ps1` will utilize the OneNote
   * `markdown_strict` (original unextended Markdown)
   * Improved headers, with title now as a `#` heading, standardized `DateTime` format for created and modified dates, and horizontal line to separate from rest of document
 * Choose whether to clear extra newlines between unordered (bullet) and ordered (numbered) list items, non-breaking spaces from blank lines, and `>` after unordered lists, which are created when converting with Pandoc
-* Choose whether to remove `\` escape symbol that are created when converting with Pandoc
+* Choose whether to remove `\` escape symbol that are created when converting with Pandoc. Mode `4` keeps only the escapes that matter and rebuilds lists that Pandoc escaped (`\- item`, `7\. item`), turns a `\` at the end of a line into a hard line break, and removes escapes that do not change the rendering
 * Choose whether to use Line Feed (`LF`) or Carriage Return + Line Feed (`CRLF`) for new lines
 * Choose whether to include a `.pdf` export alongside the `.md` file. `.md` does not preserve `InkDrawing` (i.e. overlayed drawings, highlights, pen marks) absolute positions within a page, but a `.pdf` export is a complete page snapshot that preserves `InkDrawing` absolute positions within a page.
 * Detailed logs. Run the script with `-Verbose` to see detailed logs of each page's conversion.

config.example.ps1

@@ -77,6 +77,7 @@ $keepspaces = 1
 # 1: Clear all '\' characters  - Default
 # 2: Clear all '\' characters except those preceding alphanumeric characters
 # 3: Keep '\' symbol escape
+# 4: Keep '\' only where it changes the rendering: rebuild lists and line breaks that pandoc escaped ('\- item', '7\. item', '\' at the end of a line), remove other escapes that don't matter
 $keepescape = 1
 
 # Whether to use Line Feed (LF) or Carriage Return + Line Feed (CRLF) for new lines