# # processMDSpec.py # # (c) 2025 by Andreas Kraft # License: BSD 3-Clause License. See the LICENSE file for further details. # """ This script processes markdown specification files. It handles the include statements and the front matter. It can also render the markdown content on console or output the front matter only. """ from __future__ import annotations _print = print # save the original print function from typing import Tuple, Generator import argparse from rich import markdown, print import re, sys, yaml, os from contextlib import contextmanager _frontMatter:dict = {} _includeStack:list[str] = [] @contextmanager def includeStack(filename:str) -> Generator [None, None, None]: """ Handle the include stack. This is used to detect circular includes and to keep track of the include stack. Args: filename: The name of the file being processed. Raises: Exception: If a circular include is detected. Returns: Generator: A generator that yields nothing. """ if filename in _includeStack: raise Exception(f'Circular include detected: {" -> ".join(_includeStack)} -> {filename}') _includeStack.append(filename) yield _includeStack.pop() def expandPaths(lines:list[str], currentPath:str, childPath:str) -> list[str]: """ Expand the paths in the markdown file. This means that all paths in links, images, and include statements are extended so that they would be valid paths from the root document. Args: lines: The lines of the markdown file. currentPath: The current path of the file being processed. childPath: The path of the child file being processed. Returns: list[str]: The lines of the markdown file with expanded paths. """ # Replace all relative paths in the markdown with the new path # add a path to the current path if currentPath[-1] != '/': currentPath += '/' newPath = currentPath + childPath # Remove the leading './' from the path while newPath.startswith('./'): newPath = newPath[2:] inCodeFence = False for index, line in enumerate(lines): # Ignore stuff in code fences if re.match(r'^\s*```.*', line): inCodeFence = not inCodeFence continue if inCodeFence: continue # handle the links in a line (there could be multiple links in a line) links = re.findall(r'\[([^\]]+)\]\(([^\)]+)\)', line) for linkText, linkPath in links: # Skip URLs and absolute paths if linkPath.startswith(('http://', 'https://', '/')): continue # Construct the new path by adding addedPath to the original path newLinkPath = linkPath[2:] if linkPath.startswith('./') else linkPath # Create the updated path updatedPath = f"{newPath}{linkPath}" if newPath.endswith('/') else f"{newPath}/{newLinkPath}" # Replace the original link with the updated one in the markdown line = line.replace(f'[{linkText}]({linkPath})', f'[{linkText}]({updatedPath})') # handle the include statements (there should only be one per line) includes = re.findall(r'^\s*::include{file=([^\}]+)}', line) for includePath in includes: # Construct the new path by adding addedPath to the original path includePath = includePath[2:] if includePath.startswith('./') else includePath # Create the updated path updatedPath = f'{newPath}{includePath}' if newPath.endswith('/') else f'{newPath}/{includePath}' # Replace the original include with the updated one in the markdown line = line.replace(f'::include{{file={includePath}}}', f'::include{{file={updatedPath}}}') lines[index] = line return lines def processFrontMatter(lines:list[str], args:argparse.Namespace) -> Tuple[dict, list[str]]: """ Process the front matter of a markdown file. This includes extracting the front matter information and returning it as a dictionary. Currently only YAML front matter is supported. It can be extended later. Args: lines: The lines of the markdown file. args: The command line arguments. Raises: yaml.YAMLError: If the front matter cannot be parsed as YAML. Returns: dict: The front matter information as a dictionary. list[str]: The lines of the markdown file without the front matter. """ if not lines or not lines[0].startswith('---'): return {}, lines frontMatterLines:list[str] = [] for line in lines[1:]: if re.match(r'^---\s*', line): break frontMatterLines.append(line) # Remove the front matter from the lines lines = lines[len(frontMatterLines)+2:] # Parse the front matter as YAML try: return yaml.safe_load(''.join(frontMatterLines)), lines except yaml.YAMLError as e: print(f'[red]Error parsing front matter: {e}') raise def processFile(args:argparse.Namespace) -> str: """ Handle the include statements in the markdown files. This includes processing the include statements and removing the include statements from the markdown files. Args: args: The command line arguments. Raises: Exception: If the file cannot be processed. Returns: The processed markdown content as a string. """ def handleIncludesForFile(filename:str, currentPath:str) -> str: """ Read a single markdown file and return its content. Args: filename: The name of the file to read. Raises: FileNotFoundError: If the file cannot be found. Returns: The content of the file. """ # Get the directory path from the filename dirname = os.path.dirname(filename) if dirname and not dirname.endswith('/'): dirname = dirname + '/' dirname = dirname if dirname else '.' currentPath = currentPath if currentPath else '.' filename = os.path.normpath(filename) with includeStack(filename): try: with open(filename, 'r') as f: lines = f.readlines() except FileNotFoundError: print(f'[red]File not found: {filename}') raise # Expand the paths in the markdown file # extract front matter information lines = expandPaths(lines, currentPath, dirname) fm, lines = processFrontMatter(lines, args) if fm: _frontMatter[filename] = fm if not args.doInclude: return ''.join(lines) inCodeFence = False for line in lines: # Ignore stuff code fences if re.match(r'^\s*```.*', line): inCodeFence = not inCodeFence continue if inCodeFence: continue # Check for ::include{file=...} pattern using regex at the beginning of a line match = re.search(r'^::include\{\s*file=(.*?)\s*\}', line.strip()) if match: includeFilename = match.group(1) # Read the included file and replace the include statement with its content lines[lines.index(line)] = handleIncludesForFile(includeFilename, os.path.dirname(filename)) return ''.join(lines) return handleIncludesForFile(args.document, os.path.dirname(args.document)) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Process markdown specification files.') parser.add_argument('--no-include', dest='doInclude', action='store_false', default=True, help="don't process include statements") parser.add_argument('--render-markdown', '-md', dest='renderAsMarkdown', action='store_true', help='render output as markdown') parser.add_argument('--process-frontmatter', '-fm', dest='outputFrontMatter', action='store_true', help='output front matter only') parser.add_argument('--frontmatter-only', '-fmo', dest='onlyFrontMatter', action='store_true', help='output only front matter') parser.add_argument('--verbose', '-v', action='store_true', help='print debug information to stderr.') parser.add_argument('document', type=str, help='a markdown specification document to process') args = parser.parse_args() if args.verbose: if not args.doInclude: print(f'[yellow]Skipping processing include statements', file=sys.stderr) else: print(f'[green]Processing include statements', file=sys.stderr) try: lines = processFile(args) except Exception as e: print(f'[red]Error while processing {args.document}\n{e}', file=sys.stderr) quit(1) if args.outputFrontMatter or args.onlyFrontMatter: # Collect front matter information in the output if not args.onlyFrontMatter: print('---') # The following is a workaround to keep the order of the dictionary # see https://stackoverflow.com/a/52621703 yaml.add_representer(dict, lambda self, data: yaml.representer.SafeRepresenter.represent_dict(self, data.items())) print(yaml.dump(_frontMatter, default_flow_style=False), end='') if not args.onlyFrontMatter: print('---') if not args.onlyFrontMatter: if args.renderAsMarkdown: # Render the markdown content print(markdown.Markdown(lines)) else: # Print the raw markdown content _print(lines)