From 09ad910c7e4230928385d99b1d0494448b4c8db7 Mon Sep 17 00:00:00 2001 From: Miguel Angel Reina Ortega <miguelangel.reinaortega@etsi.org> Date: Fri, 21 Jul 2023 16:27:50 +0200 Subject: [PATCH] Adding generation of tracked changes docx to the process --- .gitlab-ci.yml | 15 ++++++++------- .../addTrackedChanges.py | 19 ++++++++++++++----- .../changemarks.py | 1 + generateChangemarks/dockerfile | 7 +++++++ .../pandocFilter.py | 0 .../requirements.txt | 2 ++ .../setup.py | 0 generate_changemarks.sh | 16 +++++++++++----- pandocFilter/dockerfile | 7 ------- 9 files changed, 43 insertions(+), 24 deletions(-) rename {pandocFilter => generateChangemarks}/addTrackedChanges.py (93%) rename {pandocFilter => generateChangemarks}/changemarks.py (99%) create mode 100644 generateChangemarks/dockerfile rename {pandocFilter => generateChangemarks}/pandocFilter.py (100%) rename {pandocFilter => generateChangemarks}/requirements.txt (91%) rename {pandocFilter => generateChangemarks}/setup.py (100%) delete mode 100644 pandocFilter/dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e470ecf..d5c6a98 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,16 +19,17 @@ stages: Build pythonForPandocFilter docker image: stage: build - before_script: cd pandocFilter - script: docker build --tag pandocfilter:latest -f dockerfile . + before_script: cd generateChangemarks + script: docker build --tag generatechangemarks:latest -f dockerfile . rules: - if: $CI_COMMIT_BRANCH changes: - - pandocFilter/dockerfile - - pandocFilter/setup.py - - pandocFilter/requirements.txt - - pandocFilter/pandocFilter.py - - pandocFilter/changemarks.py + - generateChangemarks/dockerfile + - generateChangemarks/setup.py + - generateChangemarks/requirements.txt + - generateChangemarks/pandocFilter.py + - generateChangemarks/changemarks.py + - generateChangemarks/addTrackedChanges.py Word CR text: stage: generation diff --git a/pandocFilter/addTrackedChanges.py b/generateChangemarks/addTrackedChanges.py similarity index 93% rename from pandocFilter/addTrackedChanges.py rename to generateChangemarks/addTrackedChanges.py index 302b753..5fb17d8 100644 --- a/pandocFilter/addTrackedChanges.py +++ b/generateChangemarks/addTrackedChanges.py @@ -9,9 +9,11 @@ # from typing import Optional -import argparse, tempfile, zipfile +import argparse, tempfile, zipfile, sys import lxml.etree as ET +from changemarks import MR + wns = '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}' @@ -132,8 +134,7 @@ def convertChanges(fn:str, for fn in originalFileList: zip.write(f'{tmpDir}/{fn}', fn, compress_type = zipfile.ZIP_DEFLATED , compresslevel = 9) - -if __name__ == '__main__': +def main(args=None): # Parse command line arguments parser = argparse.ArgumentParser(formatter_class = argparse.ArgumentDefaultsHelpFormatter) @@ -142,11 +143,19 @@ if __name__ == '__main__': parser.add_argument('--date', '-d', action = 'store', dest = 'date', default = None, metavar = '<date>', help = 'specify the date of the changes (e.g. "2023-07-21T14:09:02")') parser.add_argument('--colourOnly', '-c', action = 'store_true', dest = 'colourOnly', default = False, help = 'only set the colour of the changes, but do not convert them to tracked changes') parser.add_argument('document', help = 'document to parse') + parser.add_argument('rootURL', help="Forge root URL") + parser.add_argument('projectID', help="Forge project ID") + parser.add_argument('mergeID', help="Merge IID") args = parser.parse_args() + mr = MR(args.projectID, args.mergeID, args.rootURL) + # Convert the changes convertChanges(args.document, args.outfile, - author = args.author, - date = args.date, + author = mr.author, + date = mr.date, colourOnly = args.colourOnly) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/pandocFilter/changemarks.py b/generateChangemarks/changemarks.py similarity index 99% rename from pandocFilter/changemarks.py rename to generateChangemarks/changemarks.py index 9d4a0a5..0bb5c7b 100644 --- a/pandocFilter/changemarks.py +++ b/generateChangemarks/changemarks.py @@ -124,6 +124,7 @@ class MR: self.web_url = self.raw_project_details['web_url'] self.raw_mr_details = fetch_json(self.api_url(f'/merge_requests/{self.mr_id}')) self.author = self.raw_mr_details['author']['name'] + self.date = self.raw_mr_details['updated_at'] self.target_branch = self.raw_mr_details['target_branch'] self.source_branch = self.raw_mr_details['source_branch'] self.title = self.raw_mr_details['title'] diff --git a/generateChangemarks/dockerfile b/generateChangemarks/dockerfile new file mode 100644 index 0000000..5c80471 --- /dev/null +++ b/generateChangemarks/dockerfile @@ -0,0 +1,7 @@ +FROM python:3.9-slim-bullseye + +ADD . /generateChangemarks/ +RUN apt-get update -y && \ + rm -rf /var/lib/apt/lists/* &&\ + pip install -e generateChangemarks/ &&\ + pip install -r generateChangemarks/requirements.txt diff --git a/pandocFilter/pandocFilter.py b/generateChangemarks/pandocFilter.py similarity index 100% rename from pandocFilter/pandocFilter.py rename to generateChangemarks/pandocFilter.py diff --git a/pandocFilter/requirements.txt b/generateChangemarks/requirements.txt similarity index 91% rename from pandocFilter/requirements.txt rename to generateChangemarks/requirements.txt index 0422266..6aa5ec7 100644 --- a/pandocFilter/requirements.txt +++ b/generateChangemarks/requirements.txt @@ -14,3 +14,5 @@ pygments==2.15.1 # via rich rich==13.4.2 # via oneM2M-markdown-to-pandoc-filter (setup.py) +requests==2.31.0 +unidiff==0.7.5 \ No newline at end of file diff --git a/pandocFilter/setup.py b/generateChangemarks/setup.py similarity index 100% rename from pandocFilter/setup.py rename to generateChangemarks/setup.py diff --git a/generate_changemarks.sh b/generate_changemarks.sh index c50a558..d410f16 100644 --- a/generate_changemarks.sh +++ b/generate_changemarks.sh @@ -1,7 +1,7 @@ #!/bin/bash FORGELIB_DOCKER_IMAGE=forge.3gpp.org:5050/tools/3gpp-scripts/forgelib:v2.5.0 -PANDOC_FILTER_DOCKER_IMAGE=pandocfilter:latest +GENERATE_CHANGEMARKS_DOCKER_IMAGE=generatechangemarks:latest DOCKER_IMAGE=pandoc/core:3.1.1.0 echo "\n------ Checking for docker image --------" @@ -21,13 +21,13 @@ echo "MERGE IID:" $3 echo "\n------ Generating change marks MD --------" #docker container run --rm -v $(pwd):/tmp/ "$DOCKER_IMAGE" forgelib-changedocs -vv --combine --diffs --preprocessor onem2m --outPath=/tmp/docs --startdelimiter "/tmp/$4" --enddelimiter "/tmp/$5" --coversheet "/tmp/$6" "$HOST_URL" "$2" "$PROJECT_NAME" "$3" -docker container run --rm -v $(pwd):/tmp/ -u $(id -u):$(id -g) "$PANDOC_FILTER_DOCKER_IMAGE" changemarks -o "/tmp/out" "$HOST_URL" "$2" "$3" +docker container run --rm -v $(pwd):/tmp/ -u $(id -u):$(id -g) "$GENERATE_CHANGEMARKS_DOCKER_IMAGE" changemarks -o "/tmp/out" "$HOST_URL" "$2" "$3" echo "\n------ Generating changemarks docx --------" for i in out/*.md ; do - DOCUMENT_NAME=$(echo $i | cut -d'/' -f 2) + DOCUMENT_NAME=$(echo $i | cut -d'/' -f 2) echo "\n------ Preparaing spec --------" - docker run --rm -v $(pwd):/tmp/ -u $(id -u):$(id -g) "$PANDOC_FILTER_DOCKER_IMAGE" pandocFilter -o "/tmp/out" "/tmp/$i" + docker run --rm -v $(pwd):/tmp/ -u $(id -u):$(id -g) "$GENERATE_CHANGEMARKS_DOCKER_IMAGE" pandocFilter -o "/tmp/out" "/tmp/$i" echo "\n------ Publishing spec --------" docker run --rm -v $(pwd):/data -u $(id -u):$(id -g) "$DOCKER_IMAGE" "/data/$i" -f markdown -t docx --reference-doc "Spec-template.docx" -o "/data/out/${DOCUMENT_NAME}.docx" done @@ -35,4 +35,10 @@ done echo "\n------ Combining docx --------" docker container run --rm -v $(pwd):/tmp/ -u $(id -u):$(id -g) "$FORGELIB_DOCKER_IMAGE" forgelib-changedocs -vv -sf "/tmp/out/" --preprocessor onem2m --outPath=/tmp/docs --startdelimiter "/tmp/$4" --enddelimiter "/tmp/$5" --coversheet "/tmp/$6" "$HOST_URL" "$2" "$3" -exit 0 +echo "\n------ Converting to tracked changes docx --------" +for i in docs/*.docx ; do + DOCUMENT_NAME=$(echo $i | cut -d'/' -f 2) + docker container run --rm -v $(pwd):/tmp/ -u $(id -u):$(id -g) "$GENERATE_CHANGEMARKS_DOCKER_IMAGE" addTrackedChanges -o "/tmp/docs/$DOCUMENT_NAME" "/tmp/$i" "$HOST_URL" "$2" "$3" +done + +exit 0 \ No newline at end of file diff --git a/pandocFilter/dockerfile b/pandocFilter/dockerfile deleted file mode 100644 index 7f39a11..0000000 --- a/pandocFilter/dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM python:3.9-slim-bullseye - -ADD . /pandocFilter/ -RUN apt-get update -y && \ - rm -rf /var/lib/apt/lists/* &&\ - pip install -e pandocFilter/ &&\ - pip install -r pandocFilter/requirements.txt -- GitLab