Script to split markdown files into clauses. Generate the mkdocs navigation...

Script to split markdown files into clauses. Generate the mkdocs navigation and copies the media files

toMkdocs/toMkdocs.py

+#
+#	toMkdocs.py
+#
+#	(c) 2024 by Andreas Kraft
+#
+#	This script converts oneM2M spec markdown file to a mkdocs compatible
+#	directory structure.
+#
+
+import argparse, re, os, shutil
+from dataclasses import dataclass
+from rich import print
+
+@dataclass
+class Clause:
+	"""	Represents a clause in the markdown file. """
+	level:int
+	title:str
+	lines:list[str]
+	onlyNav:bool = False
+
+
+_matchHeader = re.compile(r'(#+)\s+(.*)', re.IGNORECASE)
+_matchCodefence = re.compile(r'\s*```\s?.*', re.IGNORECASE)
+_match2spaceListIndention = re.compile(r'^\s{2}-', re.IGNORECASE)
+
+# TODO handle multiple nav levels (left bar) better (make conifgurable)
+# TODO Update links in the markdown files to the new structure
+
+def analyseMarkdown(filename:str) -> list[Clause]:
+	"""	Analyse the markdown file and split it into clauses.
+
+		Args:
+			filename: The name of the markdown file.
+
+		Returns:
+			The list of clauses.
+	"""
+
+	with open(filename, 'r') as file:
+		inLines = file.readlines()
+	
+	outLines:list[Clause] = [Clause(0, '', [])]
+
+	# Go through the lines and detect headers and codefences
+	inCodefence = False
+	for line in inLines:
+
+		# Detect codefences
+		if _matchCodefence.match(line):
+			inCodefence = not inCodefence
+		if inCodefence:
+			outLines[-1].lines.append(line)
+			continue
+
+		# Detect headers
+		if (m := _matchHeader.match(line)):
+			level = len(m.groups()[0])
+			clauseTitle = m.groups()[1].strip()
+			outLines.append(Clause(level, clauseTitle, []))
+
+		outLines[-1].lines.append(line)
+
+	return outLines
+
+
+def splitMarkdownDocument(clauses:list[Clause], 
+						  ignoreTitles:list[str] = [], 
+						  splitLevel:int = 1,
+						  ignoreUntilFirstHeading:bool = True) -> list[Clause]:
+	"""	Split the clauses at a certain level. This is used to create separate
+		markdown files for MkDocs.
+
+		Args:
+			clauses: The list of clauses.
+			ignoreTitles: A list of titles that should be ignored. They are not included in the output.
+			splitLevel: The level at which the clauses should be split.
+			ignoreUntilFirstHeader: Ignore all clauses until the first heading.
+		
+		Returns:
+			The list of clauses.
+	"""
+	outLines:list[Clause] = [Clause(0, '', [])]
+
+	for clause in clauses:
+		level = clause.level
+
+		# Check if the current clause should be ignored
+		if clause.title.casefold() in ignoreTitles:
+			continue
+
+		# Add a new output clause if the current clause's level is 
+  		# equal or less than the split level
+		if clause.level <= splitLevel:
+			outLines.append(Clause(level, clause.title, []))
+		
+		# Add the lines to the output clause
+		outLines[-1].lines.extend(clause.lines)
+	
+	# Remove the first clause if it has no title
+	if ignoreUntilFirstHeading:
+		while len(outLines[0].title) == 0:
+			outLines.pop(0)
+
+	return outLines
+
+
+def prepareForMkdocs(clauses:list[Clause]) -> list[Clause]:
+	"""	Prepare the clauses for MkDocs. This includes removing the heading
+		from the clauses and marking the clauses that are only for navigation.
+
+		Args:
+			clauses: The list of clauses.
+		
+		Returns:
+			The list of clauses.
+	"""
+
+	# Remove the heading from the lines. The heading is the first line
+	# in the clause. This is done because MkDocs repeats the heading when
+	# displaying the page.
+	for clause in clauses:
+		if len(clause.lines) > 0:
+			clause.lines.pop(0)
+			# Also, remove the first empty lines if they exist
+			while len(clause.lines) > 0 and clause.lines[0].strip() == '':
+				clause.lines.pop(0)
+	
+	# Mark the whole clause if it is the first AND NOT only clause
+	# for a parent clause. Then it is usually empty except the heading.
+	# We still need it for navigation, so we mark it as onlyNav
+	for clause in clauses:
+		if len(''.join(clause.lines).strip()) == 0 and clause.level > 0:
+			clause.onlyNav = True
+
+	# Repair wrong markdown for indented lines.
+	# Add 2 spaces to existing 2-space indentions
+	for clause in clauses:
+		for i, line in enumerate(clause.lines):
+			if _match2spaceListIndention.match(line):
+				clause.lines[i] = '  ' + line
+	
+	return clauses
+
+
+def writeClauses(outLines:list[Clause], filename:str, navTitle:str) -> None:
+	"""	Write the clauses to separate files and create a navigation file.
+
+		Args:
+			outLines: The list of clauses.
+			filename: The name of the original markdown file.
+			navTitle: The title of the navigation entry. This is used to determine the directories.
+	"""
+
+	# Write the files
+	# create directory first
+	os.makedirs(f'{os.path.dirname(filename)}/{navTitle}', exist_ok = True)
+	for i, f in enumerate(outLines):
+		if len(f.lines) == 0:	# ignore empty clauses
+			continue
+		# Don't write content files that are only for navigation
+		if f.onlyNav:
+			continue
+	
+		# write to single files
+		with open(f'{os.path.dirname(filename)}/{navTitle}/{i}.md', 'w') as file:
+			file.writelines(f.lines)
+			print(f'[green]File "{i}.md" written')
+
+	
+	# write nav.yml file
+	with open(f'{os.path.dirname(filename)}/_nav.yml', 'w') as file:
+		file.write(f'  - {navTitle}:\n')
+		for i, f in enumerate(outLines):
+			if f.onlyNav:
+				file.write(f"  {'  '*f.level}- '{f.title}':\n")
+			else:
+				if len(f.lines) == 0:
+					continue
+				file.write(f"  {'  '*f.level}- '{f.title}': '{navTitle}/{i}.md'\n")
+	print(f'[green]File "_nav.yml" written')
+
+
+def copyMediaFiles(filename:str, navTitle:str, mediaDirectory:str = 'media') -> None:
+	"""	Copy media files from the source directory to the target directory.
+
+		Args:
+			filename: The name of the markdown file.
+			navTitle: The title of the navigation entry.
+			mediaDirectory: The name of the media directory.
+	"""
+	sourceDirectory = f'{os.path.dirname(filename)}/{mediaDirectory}'
+	targetDirectory = f'{os.path.dirname(filename)}/{navTitle}/{mediaDirectory}'
+
+	if os.path.exists(sourceDirectory):
+		print(f'[green]Copying media files from "{sourceDirectory}" to "{targetDirectory}"')
+		shutil.copytree(sourceDirectory, targetDirectory, dirs_exist_ok = True)
+	else:
+		print(f'[red]Media directory "{sourceDirectory}" does not exist')
+
+	
+def processDocument(args:argparse.Namespace) -> None:
+	clauses = analyseMarkdown(args.document)
+	clauses = splitMarkdownDocument(clauses, [ t.casefold() for t in args.ignore_clause ], args.split_level)
+	clauses = prepareForMkdocs(clauses)
+	writeClauses(clauses, args.document, args.title)
+	copyMediaFiles(args.document, args.title, args.media_directory)
+
+
+if __name__ == '__main__':
+	parser = argparse.ArgumentParser(formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+
+	parser.add_argument('--title', '-t', metavar = 'title', required = True, help = 'mkdocs navigation tile')
+	parser.add_argument('--ignore-clause', '-i', metavar = 'clause', nargs = '+', default = [ 'Contents', 'History' ], help = 'ignore headers in the markdown document')
+	parser.add_argument('--split-level', '-sl', metavar = 'level', type = int, default = 2, help = 'split clauses on which level')
+	parser.add_argument('--media-directory', '-md', metavar = 'media-directory', default = 'media', help = 'directory where media files are stored')
+	parser.add_argument('document', type = str, help = 'a oneM2M markdown specification document to process')
+
+	args = parser.parse_args()
+
+	processDocument(args)
+	
+	
+