From 71a65a2b5b113381af41220b3b4571b1f8f8db75 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Fri, 14 Feb 2020 18:24:27 +0200 Subject: [PATCH 01/23] Add a Travis CI integration: Added a Travis CI script. Also added the requirements.txt file, after mkdocs & nose installation. --- .gitignore | 1 + .travis.yml | 9 +++++++++ requirements.txt | 10 ++++++++++ tests/codeinclude/test_plugin.py | 2 +- 4 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 .travis.yml create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 8a6468e..57fb45b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ __pycache__/ .eggs/ .pytest_cache/ .DS_Store +.venv/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..cb01e07 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,9 @@ +language: python +python: + - "3.7" +# Install the dependencies +install: + - pip install -r requirements.txt +# Run the tests +script: + - nosetests diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e6b9553 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +Click==7.0 +Jinja2==2.11.1 +livereload==2.6.1 +Markdown==3.2.1 +MarkupSafe==1.1.1 +mkdocs==1.0.4 +nose==1.3.7 +PyYAML==5.3 +six==1.14.0 +tornado==6.0.3 diff --git a/tests/codeinclude/test_plugin.py b/tests/codeinclude/test_plugin.py index 54775f4..9e7e01e 100644 --- a/tests/codeinclude/test_plugin.py +++ b/tests/codeinclude/test_plugin.py @@ -34,7 +34,7 @@ c = Config(schema=DEFAULT_SCHEMA) c["site_url"] = "http://example.org/" -PAGE_EXAMPLE = Page("", File(os.path.abspath("./fixture/text.md"), "/src", "/dest", False), c) +PAGE_EXAMPLE = Page("", File(os.path.abspath("./tests/codeinclude/fixture/text.md"), "/src", "/dest", False), c) class PluginTextCase(unittest.TestCase): From da8c831867fedbe166a5816335dcb253375ab54a Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Fri, 14 Feb 2020 19:20:43 +0200 Subject: [PATCH 02/23] Add support for any language: Use the file extension as the language class for highlight.js appears to support file extensions as language classes, see: https://github.com/highlightjs/highlight.js#supported-languages --- codeinclude/languages.py | 15 +++++++++++++++ codeinclude/plugin.py | 8 +++++++- tests/codeinclude/test_languages.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 codeinclude/languages.py create mode 100644 tests/codeinclude/test_languages.py diff --git a/codeinclude/languages.py b/codeinclude/languages.py new file mode 100644 index 0000000..c599a5e --- /dev/null +++ b/codeinclude/languages.py @@ -0,0 +1,15 @@ + + +def get_lang_class(filename: str) -> str: + """Returns the highlight.js _language class_ for the filename. + + Most MkDocs themes use the highlight.js for syntax highlighting: https://www.mkdocs.org/user-guide/styling-your-docs/#built-in-themes + + highlight.js supports file extensions as language classes. + For a table of 'Language -> Language Classes', see https://github.com/highlightjs/highlight.js#supported-languages + """ + return get_extension(filename) + + +def get_extension(filename: str) -> str: + return filename.split('.')[-1] diff --git a/codeinclude/plugin.py b/codeinclude/plugin.py index b0f3931..f1cb423 100644 --- a/codeinclude/plugin.py +++ b/codeinclude/plugin.py @@ -5,6 +5,7 @@ from mkdocs.plugins import BasePlugin from codeinclude.resolver import select +from codeinclude.languages import get_lang_class RE_START = r"""(?x) ^ @@ -44,8 +45,13 @@ def get_substitute(page, title, filename, lines, block, inside_block): ) dedented = textwrap.dedent(selected_content) + lang_code = get_lang_class(filename) + return f''' +```{lang_code} tab="{title}" +{dedented} +``` - return '\n```java tab="' + title + '"\n' + dedented + "\n```\n\n" +''' class CodeIncludePlugin(BasePlugin): diff --git a/tests/codeinclude/test_languages.py b/tests/codeinclude/test_languages.py new file mode 100644 index 0000000..42aaa43 --- /dev/null +++ b/tests/codeinclude/test_languages.py @@ -0,0 +1,29 @@ +import unittest +from codeinclude.languages import get_extension, get_lang_class + + +class MyTestCase(unittest.TestCase): + def test_get_extension_simple_name(self): + filename = 'HelloWorld.java' + expected = 'java' + self.assertEqual(get_extension(filename), expected) + + def test_get_extension_relative_name(self): + filename = './HelloWorld.java' + expected = 'java' + self.assertEqual(get_extension(filename), expected) + + def test_get_extension_dots_in_name(self): + filename = 'HelloWorld.template.java' + expected = 'java' + self.assertEqual(get_extension(filename), expected) + + def test_get_lang_class(self): + self.assertEquals('java', get_lang_class('HelloWorld.java')) + self.assertEquals('xml', get_lang_class('HelloWorld.xml')) + self.assertEquals('json', get_lang_class('HelloWorld.json')) + self.assertEquals('rs', get_lang_class('HelloWorld.rs')) + + +if __name__ == '__main__': + unittest.main() From cb29dd5d22f2740adaa2b91130ba24eef86802e6 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Mon, 17 Feb 2020 15:26:22 +0200 Subject: [PATCH 03/23] Use the language alias from pygments instead It must be more accurate than the file extension. --- codeinclude/languages.py | 23 ++++++++++++++--------- setup.py | 2 +- tests/codeinclude/test_languages.py | 23 ++++++----------------- 3 files changed, 21 insertions(+), 27 deletions(-) diff --git a/codeinclude/languages.py b/codeinclude/languages.py index c599a5e..8862ff3 100644 --- a/codeinclude/languages.py +++ b/codeinclude/languages.py @@ -1,15 +1,20 @@ +from pygments.lexers import get_lexer_for_filename +from pygments.util import ClassNotFound def get_lang_class(filename: str) -> str: - """Returns the highlight.js _language class_ for the filename. + """Returns the Pygments _language alias_ for the filename. - Most MkDocs themes use the highlight.js for syntax highlighting: https://www.mkdocs.org/user-guide/styling-your-docs/#built-in-themes + Pygments is used by codehilite, a widely used extension for code highlighting: + https://squidfunk.github.io/mkdocs-material/extensions/codehilite/ - highlight.js supports file extensions as language classes. - For a table of 'Language -> Language Classes', see https://github.com/highlightjs/highlight.js#supported-languages + The Pygments language aliases are expected to be compatible with highlight.js language classes, + which are used by some MkDocs themes: https://www.mkdocs.org/user-guide/styling-your-docs/#built-in-themes + For a table of 'Language -> Language Classes' in _highlight.js_, + see https://github.com/highlightjs/highlight.js#supported-languages """ - return get_extension(filename) - - -def get_extension(filename: str) -> str: - return filename.split('.')[-1] + try: + lexer = get_lexer_for_filename(filename) + return lexer.aliases[0] + except ClassNotFound: + return "none" diff --git a/setup.py b/setup.py index e56283a..f3fdd10 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ def read_file(fname): python_requires='>=3.6', install_requires=[ 'mkdocs>=0.17', - 'mkdocs' + 'pygments' ], classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/tests/codeinclude/test_languages.py b/tests/codeinclude/test_languages.py index 42aaa43..7479120 100644 --- a/tests/codeinclude/test_languages.py +++ b/tests/codeinclude/test_languages.py @@ -1,28 +1,17 @@ import unittest -from codeinclude.languages import get_extension, get_lang_class +from codeinclude.languages import get_lang_class class MyTestCase(unittest.TestCase): - def test_get_extension_simple_name(self): - filename = 'HelloWorld.java' - expected = 'java' - self.assertEqual(get_extension(filename), expected) - - def test_get_extension_relative_name(self): - filename = './HelloWorld.java' - expected = 'java' - self.assertEqual(get_extension(filename), expected) - - def test_get_extension_dots_in_name(self): - filename = 'HelloWorld.template.java' - expected = 'java' - self.assertEqual(get_extension(filename), expected) - def test_get_lang_class(self): self.assertEquals('java', get_lang_class('HelloWorld.java')) + self.assertEquals('python', get_lang_class('HelloWorld.py')) + self.assertEquals('csharp', get_lang_class('HelloWorld.cs')) + self.assertEquals('rust', get_lang_class('HelloWorld.rs')) + self.assertEquals('docker', get_lang_class('Dockerfile')) self.assertEquals('xml', get_lang_class('HelloWorld.xml')) + self.assertEquals('toml', get_lang_class('HelloWorld.toml')) self.assertEquals('json', get_lang_class('HelloWorld.json')) - self.assertEquals('rs', get_lang_class('HelloWorld.rs')) if __name__ == '__main__': From 78cec4fc4b09997213b5d2ea4c3cbfa5a6e46c17 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Mon, 17 Feb 2020 15:35:28 +0200 Subject: [PATCH 04/23] Remove Travis CI integration --- .travis.yml | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index cb01e07..0000000 --- a/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -language: python -python: - - "3.7" -# Install the dependencies -install: - - pip install -r requirements.txt -# Run the tests -script: - - nosetests From fe0d3f9d3cb07b8fc5a8ea5bfe78f5d3a3df2716 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Mon, 17 Feb 2020 15:46:29 +0200 Subject: [PATCH 05/23] Update requirements.txt according to the recommendations for libraries: Dependencies are specified in the setup.py, and could be resolved from here. See: https://pip.pypa.io/en/stable/user_guide/#requirements-files and https://caremad.io/posts/2013/07/setup-vs-requirement/ --- requirements.txt | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index e6b9553..ecf975e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1 @@ -Click==7.0 -Jinja2==2.11.1 -livereload==2.6.1 -Markdown==3.2.1 -MarkupSafe==1.1.1 -mkdocs==1.0.4 -nose==1.3.7 -PyYAML==5.3 -six==1.14.0 -tornado==6.0.3 +-e . \ No newline at end of file From 53631172eb8253b686e245818dd625a9c60c1e5f Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Mon, 17 Feb 2020 16:07:12 +0200 Subject: [PATCH 06/23] Allow empty title: Allow empty title for code snippets. It may be useful for short code fragments. --- codeinclude/plugin.py | 11 +++++++++-- tests/codeinclude/test_plugin.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/codeinclude/plugin.py b/codeinclude/plugin.py index f1cb423..eb88751 100644 --- a/codeinclude/plugin.py +++ b/codeinclude/plugin.py @@ -34,7 +34,14 @@ def get_substitute(page, title, filename, lines, block, inside_block): + # Compute the fence header + lang_code = get_lang_class(filename) + header = lang_code + title = title.strip() + if len(title) > 0: + header += f' tab="{title}"' + # Select the code content page_parent_dir = os.path.dirname(page.file.abs_src_path) import_path = os.path.join(page_parent_dir, filename) with open(import_path) as f: @@ -45,9 +52,9 @@ def get_substitute(page, title, filename, lines, block, inside_block): ) dedented = textwrap.dedent(selected_content) - lang_code = get_lang_class(filename) + return f''' -```{lang_code} tab="{title}" +```{header} {dedented} ``` diff --git a/tests/codeinclude/test_plugin.py b/tests/codeinclude/test_plugin.py index 9e7e01e..b1748bf 100644 --- a/tests/codeinclude/test_plugin.py +++ b/tests/codeinclude/test_plugin.py @@ -31,6 +31,17 @@ """ +EMPTY_TITLE_MARKDOWN_EXAMPLE = """ +# hello world + +some text before + +[](Foo.java) + +and some text after + +""" + c = Config(schema=DEFAULT_SCHEMA) c["site_url"] = "http://example.org/" @@ -85,3 +96,23 @@ def test_multi_tab_case(self): and some text after """).strip(), result.strip()) + + def test_empty_title_case(self): + plugin = CodeIncludePlugin() + result = plugin.on_page_markdown(EMPTY_TITLE_MARKDOWN_EXAMPLE, PAGE_EXAMPLE, dict()) + + print(result) + self.assertEqual(textwrap.dedent(""" + # hello world + + some text before + + ```java + public class Foo { + + } + ``` + + and some text after + """).strip(), + result.strip()) From 60f6e7b39729491a728fc0bf0c85a07c3e4b9702 Mon Sep 17 00:00:00 2001 From: Richard North Date: Mon, 17 Feb 2020 16:24:08 +0200 Subject: [PATCH 07/23] Import the documentation from the testcontainers project: The documentation is imported from testcontainers/testcontainers-java@fbf5092 See https://github.com/testcontainers/testcontainers-java/blob/master/docs/contributing_docs.md#codeincludes and https://github.com/testcontainers/testcontainers-java/blob/master/LICENSE --- README.md | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 46e89fa..b10b338 100644 --- a/README.md +++ b/README.md @@ -3,4 +3,79 @@ A plugin for mkdocs that allows some advanced 'includes' functionality to be used for embedded code blocks. This is effectively an extended Markdown format, but is intended to degrade gracefully when rendered with a different renderer. -This README will be extended to include examples at a later date. +## Usage + +A codeinclude block will resemble a regular markdown link surrounded by a pair of XML comments, e.g.: + + + +
<!--codeinclude-->
+[Human readable title for snippet](./relative_path_to_example_code.java) targeting_expression
+<!--/codeinclude-->
+
+ +Where `targeting_expression` could be: + +* `block:someString` or +* `inside_block:someString` + +If these are provided, the macro will seek out any line containing the token `someString` and grab the next curly brace +delimited block that it finds. `block` will grab the starting line and closing brace, whereas `inside_block` will omit +these. + +e.g., given: +```java + +public class FooService { + + public void doFoo() { + foo.doSomething(); + } + + ... + +``` + +If we use `block:doFoo` as our targeting expression, we will have the following content included into our page: + +```java +public void doFoo() { + foo.doSomething(); +} +``` + +Whereas using `inside_block:doFoo` we would just have the inner content of the method included: + +```java +foo.doSomething(); +``` + +Note that: + +* Any code included will be have its indentation reduced +* Every line in the source file will be searched for an instance of the token (e.g. `doFoo`). If more than one line + includes that token, then potentially more than one block could be targeted for inclusion. It is advisable to use a + specific, unique token to avoid unexpected behaviour. + +When we wish to include a section of code that does not naturally appear within braces, we can simply insert our token, +with matching braces, in a comment. +While a little ugly, this has the benefit of working in any context and is easy to understand. +For example: + +```java +public class FooService { + + public void boringMethod() { + doSomethingBoring(); + + // doFoo { + doTheThingThatWeActuallyWantToShow(); + // } + } + + +``` \ No newline at end of file From 17249624c34cb658494c5c2c600d0304a8fe7485 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Mon, 17 Feb 2020 16:29:14 +0200 Subject: [PATCH 08/23] Slightly extend the docs. --- README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b10b338..3dc366b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ This is effectively an extended Markdown format, but is intended to degrade grac ## Usage -A codeinclude block will resemble a regular markdown link surrounded by a pair of XML comments, e.g.: +A codeinclude block resembles a regular markdown link surrounded by a pair of XML comments, e.g.: +[foo](Foo.java) lines:1 + +and some text after + +""" + +MARKDOWN_EXAMPLE_SELECTOR_ON_NEXT_LINE = """ +# hello world + +some text before + +[foo](Foo.java) +lines:1 + +and some text after + +""" + MULTI_TAB_MARKDOWN_EXAMPLE = """ # hello world @@ -39,9 +62,9 @@ class PluginTextCase(unittest.TestCase): - def test_simple_case(self): + def test_simple_case_no_selector(self): plugin = CodeIncludePlugin() - result = plugin.on_page_markdown(MARKDOWN_EXAMPLE, PAGE_EXAMPLE, dict()) + result = plugin.on_page_markdown(MARKDOWN_EXAMPLE_NO_SELECTOR, PAGE_EXAMPLE, dict()) print(result) self.assertEqual(textwrap.dedent(""" @@ -59,6 +82,44 @@ def test_simple_case(self): """).strip(), result.strip()) + def test_simple_case_selector_on_same_line(self): + plugin = CodeIncludePlugin() + result = plugin.on_page_markdown(MARKDOWN_EXAMPLE_SELECTOR_ON_SAME_LINE, PAGE_EXAMPLE, dict()) + + print(result) + self.assertEqual(textwrap.dedent(""" + # hello world + + some text before + + ```java tab=\"foo\" + public class Foo { + + ``` + + and some text after + """).strip(), + result.strip()) + + def test_simple_case_selector_on_next_line(self): + plugin = CodeIncludePlugin() + result = plugin.on_page_markdown(MARKDOWN_EXAMPLE_SELECTOR_ON_NEXT_LINE, PAGE_EXAMPLE, dict()) + + print(result) + self.assertEqual(textwrap.dedent(""" + # hello world + + some text before + + ```java tab=\"foo\" + public class Foo { + + ``` + + and some text after + """).strip(), + result.strip()) + def test_multi_tab_case(self): plugin = CodeIncludePlugin() result = plugin.on_page_markdown(MULTI_TAB_MARKDOWN_EXAMPLE, PAGE_EXAMPLE, dict()) From 765c31df945eb49187ba6c4b82e308df94ca54d6 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Wed, 19 Feb 2020 17:43:12 +0200 Subject: [PATCH 15/23] Add find_code_include_blocks (with two inner loops) --- codeinclude/plugin.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/codeinclude/plugin.py b/codeinclude/plugin.py index b0f3931..9c7b8c9 100644 --- a/codeinclude/plugin.py +++ b/codeinclude/plugin.py @@ -2,6 +2,8 @@ import os import shlex import textwrap +from dataclasses import dataclass +from typing import List from mkdocs.plugins import BasePlugin from codeinclude.resolver import select @@ -48,6 +50,32 @@ def get_substitute(page, title, filename, lines, block, inside_block): return '\n```java tab="' + title + '"\n' + dedented + "\n```\n\n" +@dataclass +class CodeIncludeBlock(object): + first_line_index: int + last_line_index: int + content: str + + +def find_code_include_blocks(markdown: str) -> List[CodeIncludeBlock]: + ci_blocks = list() + index = 0 + lines = markdown.splitlines() + while index < len(lines): + if re.match(RE_START, lines[index]): + # Start of the ci block + start = index + index += 1 + # Find the end of the ci block + while index < len(lines) and not re.match(RE_END, lines[index]): + index += 1 + if index < len(lines): + last = index + content = '\n'.join(lines[start:last+1]) + ci_blocks.append(CodeIncludeBlock(start, last, content)) + return ci_blocks + + class CodeIncludePlugin(BasePlugin): def on_page_markdown(self, markdown, page, config, site_navigation=None, **kwargs): "Provide a hook for defining functions from an external module" From facd249e8753fd6f77f8a3c29fc3f931fb03772b Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Wed, 19 Feb 2020 18:19:00 +0200 Subject: [PATCH 16/23] Rewrite find_code_include_blocks (with a flag and single loop) --- codeinclude/plugin.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/codeinclude/plugin.py b/codeinclude/plugin.py index 9c7b8c9..d2e2fdb 100644 --- a/codeinclude/plugin.py +++ b/codeinclude/plugin.py @@ -3,7 +3,7 @@ import shlex import textwrap from dataclasses import dataclass -from typing import List +from typing import Dict from mkdocs.plugins import BasePlugin from codeinclude.resolver import select @@ -57,22 +57,19 @@ class CodeIncludeBlock(object): content: str -def find_code_include_blocks(markdown: str) -> List[CodeIncludeBlock]: - ci_blocks = list() - index = 0 +def find_code_include_blocks(markdown: str) -> Dict[int, CodeIncludeBlock]: + ci_blocks = dict() + first = -1 lines = markdown.splitlines() - while index < len(lines): + for index, line in enumerate(lines): if re.match(RE_START, lines[index]): - # Start of the ci block - start = index - index += 1 - # Find the end of the ci block - while index < len(lines) and not re.match(RE_END, lines[index]): - index += 1 - if index < len(lines): - last = index - content = '\n'.join(lines[start:last+1]) - ci_blocks.append(CodeIncludeBlock(start, last, content)) + first = index + continue + if re.match(RE_END, lines[index]): + last = index + content = '\n'.join(lines[first:last + 1]) + ci_blocks[first] = CodeIncludeBlock(first, last, content) + first = -1 return ci_blocks From 536fdab2eecbf8090975066848c08f7e853e1a3b Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Wed, 19 Feb 2020 18:56:58 +0200 Subject: [PATCH 17/23] Add test with no includes --- tests/codeinclude/test_plugin.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/codeinclude/test_plugin.py b/tests/codeinclude/test_plugin.py index 3ef28c3..40e4428 100644 --- a/tests/codeinclude/test_plugin.py +++ b/tests/codeinclude/test_plugin.py @@ -8,6 +8,13 @@ from codeinclude.plugin import CodeIncludePlugin +MARKDOWN_EXAMPLE_NO_INCLUDES = """ +# hello world + +some text before + +""" + MARKDOWN_EXAMPLE_NO_SELECTOR = """ # hello world @@ -62,6 +69,13 @@ class PluginTextCase(unittest.TestCase): + def test_no_includes(self): + plugin = CodeIncludePlugin() + result = plugin.on_page_markdown(MARKDOWN_EXAMPLE_NO_INCLUDES, PAGE_EXAMPLE, dict()) + + self.assertEqual(MARKDOWN_EXAMPLE_NO_INCLUDES.strip(), + result.strip()) + def test_simple_case_no_selector(self): plugin = CodeIncludePlugin() result = plugin.on_page_markdown(MARKDOWN_EXAMPLE_NO_SELECTOR, PAGE_EXAMPLE, dict()) From 108ec93ff587c2155c427bf4b03429b1e61e0a66 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Wed, 19 Feb 2020 18:57:50 +0200 Subject: [PATCH 18/23] Support targeting expression on next line --- codeinclude/plugin.py | 162 ++++++++++++++++++++++++------------------ 1 file changed, 91 insertions(+), 71 deletions(-) diff --git a/codeinclude/plugin.py b/codeinclude/plugin.py index d2e2fdb..e1cfdc2 100644 --- a/codeinclude/plugin.py +++ b/codeinclude/plugin.py @@ -3,7 +3,7 @@ import shlex import textwrap from dataclasses import dataclass -from typing import Dict +from typing import List from mkdocs.plugins import BasePlugin from codeinclude.resolver import select @@ -24,30 +24,23 @@ $ """ -RE_SNIPPET = r"""(?x) +RE_SNIPPET = r"""(?xm) ^ (?P\s*) \[(?P[^\]]*)\]\((?P<filename>[^)]+)\) - ([\t ]+(?P<params>.*))? + ([\t\n ]+(?P<params>[\w:-]+))? (?P<ignored_trailing_space>\s*) $ """ -def get_substitute(page, title, filename, lines, block, inside_block): - - page_parent_dir = os.path.dirname(page.file.abs_src_path) - import_path = os.path.join(page_parent_dir, filename) - with open(import_path) as f: - content = f.read() - - selected_content = select( - content, lines=lines, block=block, inside_block=inside_block - ) - - dedented = textwrap.dedent(selected_content) +class CodeIncludePlugin(BasePlugin): + def on_page_markdown(self, markdown, page, config, site_navigation=None, **kwargs): + "Provide a hook for defining functions from an external module" - return '\n```java tab="' + title + '"\n' + dedented + "\n```\n\n" + blocks = find_code_include_blocks(markdown) + substitutes = get_substitutes(blocks, page) + return substitute(markdown, substitutes) @dataclass @@ -57,69 +50,96 @@ class CodeIncludeBlock(object): content: str -def find_code_include_blocks(markdown: str) -> Dict[int, CodeIncludeBlock]: - ci_blocks = dict() +def find_code_include_blocks(markdown: str) -> List[CodeIncludeBlock]: + ci_blocks = list() first = -1 + in_block = False lines = markdown.splitlines() for index, line in enumerate(lines): if re.match(RE_START, lines[index]): + if in_block: + raise ValueError(f"Found two consecutive code-include starts: at lines {first} and {index}") first = index - continue - if re.match(RE_END, lines[index]): + in_block = True + elif re.match(RE_END, lines[index]): + if not in_block: + raise ValueError(f"Found code-include end without preceding start at line {index}") last = index content = '\n'.join(lines[first:last + 1]) - ci_blocks[first] = CodeIncludeBlock(first, last, content) - first = -1 + ci_blocks.append(CodeIncludeBlock(first, last, content)) + in_block = False return ci_blocks -class CodeIncludePlugin(BasePlugin): - def on_page_markdown(self, markdown, page, config, site_navigation=None, **kwargs): - "Provide a hook for defining functions from an external module" +@dataclass +class Replacement(object): + first_line_index: int + last_line_index: int + content: str + + +def get_substitutes(blocks: List[CodeIncludeBlock], page) -> List[Replacement]: + replacements = list() + for ci_block in blocks: + replacement_content = "" + for snippet_match in re.finditer(RE_SNIPPET, ci_block.content): + title = snippet_match.group("title") + filename = snippet_match.group("filename") + indent = snippet_match.group("leading_space") + raw_params = snippet_match.group("params") + + if raw_params: + params = dict(token.split(":") for token in shlex.split(raw_params)) + lines = params.get("lines", "") + block = params.get("block", "") + inside_block = params.get("inside_block", "") + else: + lines = "" + block = "" + inside_block = "" + + code_block = get_substitute( + page, title, filename, lines, block, inside_block + ) + # re-indent + code_block = re.sub("^", indent, code_block, flags=re.MULTILINE) + + replacement_content += code_block + replacements.append(Replacement(ci_block.first_line_index, ci_block.last_line_index, replacement_content)) + return replacements + + +def get_substitute(page, title, filename, lines, block, inside_block): + page_parent_dir = os.path.dirname(page.file.abs_src_path) + import_path = os.path.join(page_parent_dir, filename) + with open(import_path) as f: + content = f.read() + + selected_content = select( + content, lines=lines, block=block, inside_block=inside_block + ) + + dedented = textwrap.dedent(selected_content) + + return '\n```java tab="' + title + '"\n' + dedented + "\n```\n\n" + - active = False - results = "" - for line in markdown.splitlines(): - boundary = False - - # detect end - if active and re.match(RE_END, line): - active = False - boundary = True - - # handle each line of a codeinclude zone - if active: - snippet_match = re.match(RE_SNIPPET, line) - if snippet_match: - title = snippet_match.group("title") - filename = snippet_match.group("filename") - indent = snippet_match.group("leading_space") - raw_params = snippet_match.group("params") - - if raw_params: - params = dict(token.split(":") for token in shlex.split(raw_params)) - lines = params.get("lines", "") - block = params.get("block", "") - inside_block = params.get("inside_block", "") - else: - lines = "" - block = "" - inside_block = "" - - code_block = get_substitute( - page, title, filename, lines, block, inside_block - ) - # re-indent - code_block = re.sub("^", indent, code_block, flags=re.MULTILINE) - results += code_block - - # detect start - if re.match(RE_START, line): - active = True - boundary = True - - # outside a codeinclude zone and ignoring the boundaries - if not active and not boundary: - results += line + "\n" - - return results +def substitute(markdown: str, substitutes: List[Replacement]) -> str: + substitutes_by_first_line = dict() + # Index substitutes by the first line + for s in substitutes: + substitutes_by_first_line[s.first_line_index] = s + + # Perform substitutions + result = "" + index = 0 + markdown_lines = markdown.splitlines() + while index < len(markdown_lines): + if index in substitutes_by_first_line.keys(): + substitute = substitutes_by_first_line[index] + result += substitute.content + index = substitute.last_line_index + else: + result += markdown_lines[index] + "\n" + index += 1 + return result From 7fa82ec87a7b4373c630fdd6aecb98ff07bb9d4f Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev <dmytro.tymofeev@xdev.re> Date: Thu, 20 Feb 2020 12:56:42 +0200 Subject: [PATCH 19/23] Add some comments --- codeinclude/plugin.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/codeinclude/plugin.py b/codeinclude/plugin.py index e1cfdc2..bcdda64 100644 --- a/codeinclude/plugin.py +++ b/codeinclude/plugin.py @@ -133,13 +133,15 @@ def substitute(markdown: str, substitutes: List[Replacement]) -> str: # Perform substitutions result = "" index = 0 - markdown_lines = markdown.splitlines() - while index < len(markdown_lines): + lines = markdown.splitlines() + while index < len(lines): if index in substitutes_by_first_line.keys(): + # Replace the codeinclude fragment starting at this line substitute = substitutes_by_first_line[index] result += substitute.content index = substitute.last_line_index else: - result += markdown_lines[index] + "\n" + # Keep the input line + result += lines[index] + "\n" index += 1 return result From 05bb3d4da0f523c2d2382f7a5af4bdba83f05059 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev <dmytro.tymofeev@xdev.re> Date: Fri, 21 Feb 2020 10:41:13 +0200 Subject: [PATCH 20/23] Add a failing test for code containing right-curly --- tests/codeinclude/fixture/Curly.java | 3 +++ tests/codeinclude/test_plugin.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/codeinclude/fixture/Curly.java diff --git a/tests/codeinclude/fixture/Curly.java b/tests/codeinclude/fixture/Curly.java new file mode 100644 index 0000000..04101d4 --- /dev/null +++ b/tests/codeinclude/fixture/Curly.java @@ -0,0 +1,3 @@ +public class Curly { + public static String RIGHT_CURLY_REGEX = "\\}"; +} \ No newline at end of file diff --git a/tests/codeinclude/test_plugin.py b/tests/codeinclude/test_plugin.py index bca1396..5d05031 100644 --- a/tests/codeinclude/test_plugin.py +++ b/tests/codeinclude/test_plugin.py @@ -72,6 +72,14 @@ """ +MARKDOWN_EXAMPLE_RIGHT_CURLY = """ +# hello world + +<!--codeinclude--> +[Curly](Curly.java) block:Curly +<!--/codeinclude--> +""" + c = Config(schema=DEFAULT_SCHEMA) c["site_url"] = "http://example.org/" @@ -107,6 +115,24 @@ def test_simple_case_no_selector(self): """).strip(), result.strip()) + def test_simple_case_right_curly_inside_block(self): + plugin = CodeIncludePlugin() + result = plugin.on_page_markdown(MARKDOWN_EXAMPLE_RIGHT_CURLY, PAGE_EXAMPLE, dict()) + + print(result) + self.assertEqual(textwrap.dedent(r""" + # hello world + + + ```java tab="Curly" + public class Curly { + public static String RIGHT_CURLY_REGEX = "\\}"; + } + + ``` + """).strip(), + result.strip()) + def test_simple_case_selector_on_same_line(self): plugin = CodeIncludePlugin() result = plugin.on_page_markdown(MARKDOWN_EXAMPLE_SELECTOR_ON_SAME_LINE, PAGE_EXAMPLE, dict()) From 2979ea4ac446aac13772f7bbcb94f4c4a6d34305 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev <dmytro.tymofeev@xdev.re> Date: Thu, 12 Mar 2020 20:11:58 +0200 Subject: [PATCH 21/23] Fix inclusion with brackets on the same line --- codeinclude/resolver.py | 5 +- tests/codeinclude/test_plugin.py | 1 + tests/codeinclude/test_resolver.py | 79 +++++++++++++++++++++++------- 3 files changed, 66 insertions(+), 19 deletions(-) diff --git a/codeinclude/resolver.py b/codeinclude/resolver.py index b1610c9..29a804b 100644 --- a/codeinclude/resolver.py +++ b/codeinclude/resolver.py @@ -55,9 +55,10 @@ def select( delim_count -= line.count("}") - if delim_count > 0 and not first_line_of_block: + if not first_line_of_block: delim_count += line.count("{") - selected_lines.append(i) + if delim_count > 0: + selected_lines.append(i) if from_token and to_token: i = 0 diff --git a/tests/codeinclude/test_plugin.py b/tests/codeinclude/test_plugin.py index 5d05031..78aa01d 100644 --- a/tests/codeinclude/test_plugin.py +++ b/tests/codeinclude/test_plugin.py @@ -115,6 +115,7 @@ def test_simple_case_no_selector(self): """).strip(), result.strip()) + @unittest.skip("https://github.com/rnorth/mkdocs-codeinclude-plugin/issues/13") def test_simple_case_right_curly_inside_block(self): plugin = CodeIncludePlugin() result = plugin.on_page_markdown(MARKDOWN_EXAMPLE_RIGHT_CURLY, PAGE_EXAMPLE, dict()) diff --git a/tests/codeinclude/test_resolver.py b/tests/codeinclude/test_resolver.py index 25a7d6b..5d40886 100644 --- a/tests/codeinclude/test_resolver.py +++ b/tests/codeinclude/test_resolver.py @@ -1,3 +1,6 @@ +import textwrap +import unittest + from codeinclude.resolver import select CODE_BLOCK_EXAMPLE = """ @@ -8,20 +11,62 @@ this is a trailing line """ -def test_lines(): - result = select(CODE_BLOCK_EXAMPLE, lines="2,6") - assert result == ("this is the first line\n" - "\n" - "⋯\n" - "\n" - "this is a trailing line\n") - -def test_inside_block(): - result = select(CODE_BLOCK_EXAMPLE, inside_block="blockstarter") - assert result == " block content\n" - -def test_whole_block(): - result = select(CODE_BLOCK_EXAMPLE, block="blockstarter") - assert result == ("blockstarter {\n" - " block content\n" - "}\n") \ No newline at end of file + +class ResolverTest(unittest.TestCase): + def test_lines(self): + result = select(CODE_BLOCK_EXAMPLE, lines="2,6") + self.assertEquals(("this is the first line\n" + "\n" + "⋯\n" + "\n" + "this is a trailing line\n"), + result) + + def test_inside_block(self): + result = select(CODE_BLOCK_EXAMPLE, inside_block="blockstarter") + self.assertEquals(" block content\n", result) + + def test_whole_block(self): + result = select(CODE_BLOCK_EXAMPLE, block="blockstarter") + self.assertEquals(("blockstarter {\n" + " block content\n" + "}\n"), + result) + + def test_inside_block_content_on_last_line(self): + result = select( + textwrap.dedent( + """ + foo { + if (true) { + bar(); + } } + /* The line above contains both the closing curly bracket for `if` and for `foo` */ + """), + inside_block="foo") + self.assertEquals((" if (true) {\n" + " bar();\n"), + result) + + def test_inside_block_curly_on_same_line(self): + result = select( + textwrap.dedent( + """ + foo { + /* {} */ + } + """), + inside_block="foo") + self.assertEquals(" /* {} */\n", result) + + def test_inside_block_multiple_curly_on_same_line(self): + result = select( + textwrap.dedent( + """ + // + foo { + /* {} {@code bar} {@link baz} */ + } + """), + inside_block="foo") + self.assertEquals(" /* {} {@code bar} {@link baz} */\n", result) From 9916bd55d03f05b9c5b87907404911ad9c4f553f Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev <dmytro.tymofeev@xdev.re> Date: Thu, 12 Mar 2020 20:18:08 +0200 Subject: [PATCH 22/23] Add a similar test for block --- tests/codeinclude/test_resolver.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/codeinclude/test_resolver.py b/tests/codeinclude/test_resolver.py index 5d40886..9ed1ab9 100644 --- a/tests/codeinclude/test_resolver.py +++ b/tests/codeinclude/test_resolver.py @@ -33,6 +33,22 @@ def test_whole_block(self): "}\n"), result) + def test_block_curly_on_same_line(self): + result = select( + textwrap.dedent( + """ + /* Before foo */ + foo { + /* {} {@code Bar} */ + } + /* After foo */ + """), + block="foo") + self.assertEquals(("foo {\n" + " /* {} {@code Bar} */\n" + "}\n"), + result) + def test_inside_block_content_on_last_line(self): result = select( textwrap.dedent( From 8540916fc5825cfe31aa263cfd4e8092a05525c1 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev <dmytro.tymofeev@xdev.re> Date: Fri, 13 Mar 2020 13:25:59 +0200 Subject: [PATCH 23/23] Actually fix the inclusion inside_block: The fix in 2979ea4ac446aac13772f7bbcb94f4c4a6d34305 was incomplete: see the added tests. --- codeinclude/resolver.py | 25 ++++-- tests/codeinclude/test_resolver.py | 124 +++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 8 deletions(-) diff --git a/codeinclude/resolver.py b/codeinclude/resolver.py index 29a804b..0a1bb13 100644 --- a/codeinclude/resolver.py +++ b/codeinclude/resolver.py @@ -43,22 +43,31 @@ def select( delim_count -= line.count("}") if inside_block: - i = 0 delim_count = 0 - for line in text.splitlines(): + inside_matching = False + for line_number, line in enumerate(text.splitlines(), start=1): first_line_of_block = False - i = i + 1 + # Detect the block beginning if inside_block in line and delim_count <= 0: delim_count = 0 first_line_of_block = True - delim_count += line.count("{") + inside_matching = True + # Don't process lines that are outside the matching block + if not inside_matching: + continue + + # Count the brackets in the line + delim_count += line.count("{") delim_count -= line.count("}") - if not first_line_of_block: - delim_count += line.count("{") - if delim_count > 0: - selected_lines.append(i) + # If we closed the opening bracket (= dropped below 0), the matching block has ended + if delim_count <= 0: + inside_matching = False + + # Append the lines inside the matching block, skipping the first matching + if inside_matching and not first_line_of_block: + selected_lines.append(line_number) if from_token and to_token: i = 0 diff --git a/tests/codeinclude/test_resolver.py b/tests/codeinclude/test_resolver.py index 9ed1ab9..f9615e2 100644 --- a/tests/codeinclude/test_resolver.py +++ b/tests/codeinclude/test_resolver.py @@ -86,3 +86,127 @@ def test_inside_block_multiple_curly_on_same_line(self): """), inside_block="foo") self.assertEquals(" /* {} {@code bar} {@link baz} */\n", result) + + def test_inside_block_in_a_block(self): + result = select( + textwrap.dedent( + """ + {{{ + foo { + /* inside foo */ + } + }}} + """), + inside_block="foo") + self.assertEquals(" /* inside foo */\n", result) + + def test_inside_block_contains_keyword(self): + result = select( + textwrap.dedent( + """ + /* Some code before {} */ + first { + /* first */ + first(); + if (first()) { + first(); + } else { + first(); + } + } + /* Some code after {} */ + """), + inside_block="first") + self.maxDiff = None + self.assertEquals( +""" /* first */ + first(); + if (first()) { + first(); + } else { + first(); + } +""", + result) + + def test_inside_block_nested_matching_blocks(self): + result = select( + textwrap.dedent( + """ + /* Some code before {} */ + first { + first { + first { + /* The most deeply nested. */ + } + } + } + /* Some code after {} */ + """), + inside_block="first") + self.maxDiff = None + self.assertEquals( +""" first { + first { + /* The most deeply nested. */ + } + } +""", + result) + + def test_inside_block_multiple_blocks_first(self): + result = select( + textwrap.dedent( + """ + /* Some code before {} */ + first { + /* inside first */ + } + /* Some code in between */ + second { + /* inside second */ + } + /* Some code after {} */ + """), + inside_block="first") + self.maxDiff = None + self.assertEquals(" /* inside first */\n", result) + + def test_inside_block_multiple_blocks_second(self): + result = select( + textwrap.dedent( + """ + /* Some code before {} */ + first { + /* inside first */ + } + /* Some code in between */ + second { + /* inside second */ + } + /* Some code after {} */ + """), + inside_block="second") + self.maxDiff = None + self.assertEquals(" /* inside second */\n", result) + + def test_inside_block_several_matching_blocks(self): + result = select( + textwrap.dedent( + """ + /* Some code before {} */ + matching_block 1 { + /* inside first */ + } + /* Some code in between */ + matching_block 2 { + /* inside second */ + } + /* Some code after {} */ + """), + inside_block="matching_block") + self.maxDiff = None + self.assertEquals((" /* inside first */\n" + "\n⋯\n\n" + " /* inside second */\n"), + result)