Skip to content
Snippets Groups Projects
Commit ef76da8a authored by Miguel Angel Reina Ortega's avatar Miguel Angel Reina Ortega
Browse files

Restructuring of scripts and tools

parent f0357e37
No related branches found
No related tags found
No related merge requests found
Pipeline #2248 failed
Showing
with 138 additions and 59 deletions
...@@ -14,30 +14,57 @@ stages: ...@@ -14,30 +14,57 @@ stages:
- build - build
- upgrade - upgrade
Build pythonForPandocFilter docker image: Build generateCR docker image:
stage: build stage: build
before_script: cd generateChangemarks script: docker build --tag forge.etsi.org:5050/cti/tools/generatecr:$CI_COMMIT_BRANCH -f generateCR/dockerfile .
script: docker build --tag generatechangemarks:$CI_COMMIT_BRANCH -f dockerfile .
rules: rules:
- if: $CI_COMMIT_BRANCH && $CI_PROJECT_NAME == "scripts" - if: $CI_COMMIT_BRANCH && $CI_PROJECT_NAME == "scripts"
changes: changes:
- generateChangemarks/dockerfile - generateCR/dockerfile
- generateChangemarks/setup.py - generateCR/setup.py
- generateChangemarks/requirements.txt - generateCR/requirements.txt
- generateChangemarks/pandocFilter.py - generateCR/changemarks.py
- generateChangemarks/changemarks.py - generateCR/addTrackedChanges.py
- generateChangemarks/addTrackedChanges.py - generateChangemarks/checking_conflicts.sh
- generateChangemarks/generateTOC.py
- generateChangemarks/svg2png.py Build generateBaseline docker image:
stage: build
script: docker build --tag forge.etsi.org:5050/cti/tools/generatebaseline:$CI_COMMIT_BRANCH -f generateBaseline/dockerfile .
rules:
- if: $CI_COMMIT_BRANCH && $CI_PROJECT_NAME == "scripts"
changes:
- generateBaseline/dockerfile
- generateBaseline/setup.py
- generateBaseline/requirements.txt
- generateBaseline/pandocFilter.py
- generateBaseline/generateTOC.py
- generateBaseline/svg2png.py
Build generateSpecWebSite docker image:
stage: build
script: docker build --tag forge.etsi.org:5050/cti/tools/generatespecwebsite:$CI_COMMIT_BRANCH -f generateSpecWebSite/dockerfile .
rules:
- if: $CI_COMMIT_BRANCH && $CI_PROJECT_NAME == "scripts"
changes:
- generateSpecWebSite/dockerfile
- generateSpecWebSite/setup.py
- generateSpecWebSite/requirements.txt
- generateSpecWebSite/gridTableFilter.py
- generateSpecWebSite/gridTableTools.py
- generateSpecWebSite/indexDownload.md
- generateSpecWebSite/markdownTools.py
- generateSpecWebSite/regexMatches.py
- generateSpecWebSite/spec_on_pages.sh
- generateSpecWebSite/toMkdocs.py
Build Pandoc docker image: Build Pandoc docker image:
stage: build stage: build
before_script: cd generateChangemarks before_script: cd generateBaseline
script: docker build --tag pandoc:$CI_COMMIT_BRANCH -f dockerfile.pandoc . script: docker build --tag pandoc:$CI_COMMIT_BRANCH -f dockerfile.pandoc .
rules: rules:
- if: $CI_COMMIT_BRANCH && $CI_PROJECT_NAME == "scripts" - if: $CI_COMMIT_BRANCH && $CI_PROJECT_NAME == "scripts"
changes: changes:
- generateChangemarks/dockerfile.pandoc - generateBaseline/dockerfile.pandoc
Upgrade index for pages: Upgrade index for pages:
stage: upgrade stage: upgrade
......
FROM python:3.9-slim-bullseye FROM python:3.9-slim-bullseye
ADD . /generateChangemarks/ ADD /generateChangemarks/ /generateChangemarks/
RUN chmod +x generateChangemarks/checking_conflicts.sh
RUN apt-get update -y && \ RUN apt-get update -y && \
apt-get install -y libcairo2 && \ apt-get install -y libcairo2 && \
rm -rf /var/lib/apt/lists/* &&\ rm -rf /var/lib/apt/lists/* &&\
......
File moved
File moved
File moved
File moved
File moved
FROM python:3.9-slim-bullseye
ADD /generateCR/ /generateCR/
RUN chmod +x generateCR/checking_conflicts.sh
RUN apt-get update -y && \
# apt-get install -y libcairo2 && \
rm -rf /var/lib/apt/lists/* &&\
pip install -e generateCR/ &&\
pip install -r generateCR/requirements.txt
#
# This file is autogenerated by pip-compile with Python 3.10
# by the following command:
#
# pip-compile
#
lxml==4.9.3
# via oneM2M-markdown-to-pandoc-filter (setup.py)
markdown-it-py==3.0.0
# via rich
mdurl==0.1.2
# via markdown-it-py
pygments==2.15.1
# via rich
rich==13.4.2
# via oneM2M-markdown-to-pandoc-filter (setup.py)
requests==2.31.0
unidiff==0.7.5
cairosvg==2.7.1
\ No newline at end of file
from setuptools import setup, find_packages
setup(
name='oneM2M markdown to pandoc filter',
version='0.0.1',
author='Andreas Kraft, Miguel Angel Reina Ortega',
author_email='an.kraft@gmail.com , miguelangel.reinaortega@etsi.org',
description='Convert oneM2M Markdown to Pandoc input',
packages=find_packages(),
install_requires=[
'rich',
'lxml',
],
entry_points= {
'console_scripts' : ['pandocFilter=pandocFilter:main',
'changemarks=changemarks:main',
'addTrackedChanges=addTrackedChanges:main',
'generateTOC=generateTOC:main',
'svg2png=svg2png:main',
]
}
)
...@@ -23,6 +23,7 @@ stages: ...@@ -23,6 +23,7 @@ stages:
Checking conflicts: Checking conflicts:
stage: checking stage: checking
image: forge.etsi.org:5050/cti/tools/generatecr:restructure
rules: rules:
- if: $CI_MERGE_REQUEST_TITLE !~ /v.*_baseline$/ && $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_MILESTONE - if: $CI_MERGE_REQUEST_TITLE !~ /v.*_baseline$/ && $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_MILESTONE
when: always when: always
...@@ -33,19 +34,18 @@ Checking conflicts: ...@@ -33,19 +34,18 @@ Checking conflicts:
variables: variables:
MERGE_REQUEST_MILESTONE: "all" MERGE_REQUEST_MILESTONE: "all"
allow_failure: true allow_failure: true
image: python:3.9.18-slim-bullseye
tags: tags:
- docker - docker
before_script: before_script:
# Installation of required software # Installation of required software
- apt-get update -qq && apt-get -qq install -y git curl jq > /dev/null - apt-get update -qq && apt-get -qq install -y git curl jq > /dev/null
- | #- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/checking_conflicts%2Esh/raw?ref=master" >> checking_conflicts.sh # curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/checking_conflicts%2Esh/raw?ref=master" >> checking_conflicts.sh
- chmod +x checking_conflicts.sh #- chmod +x checking_conflicts.sh
- git clone "https://$CI_SERVER_HOST/$CI_PROJECT_PATH.git" ${CI_PROJECT_NAME} - git clone "https://$CI_SERVER_HOST/$CI_PROJECT_PATH.git" ${CI_PROJECT_NAME}
script: script:
- echo 'Checking conflicts' - echo 'Checking conflicts'
- ./checking_conflicts.sh ${CI_API_V4_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${MERGE_REQUEST_MILESTONE} ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME} ${CI_MERGE_REQUEST_IID} ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} ${CI_PROJECT_NAME} "$GITLAB_USER_NAME" $GITLAB_USER_EMAIL $MIKE_ACCESS_TOKEN - ./generateCR/checking_conflicts.sh ${CI_API_V4_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${MERGE_REQUEST_MILESTONE} ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME} ${CI_MERGE_REQUEST_IID} ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} ${CI_PROJECT_NAME} "$GITLAB_USER_NAME" $GITLAB_USER_EMAIL $MIKE_ACCESS_TOKEN
artifacts: artifacts:
when: on_failure when: on_failure
paths: paths:
...@@ -61,16 +61,16 @@ Word CR: ...@@ -61,16 +61,16 @@ Word CR:
when: on_success when: on_success
before_script: before_script:
- | - |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/generate_changemarks%2Esh/raw?ref=master" >> generate_changemarks.sh curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/generateCR%2Fgenerate_changemarks%2Esh/raw?ref=master" >> generate_changemarks.sh
- chmod +x generate_changemarks.sh - chmod +x generate_changemarks.sh
- | - |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_start%2Edocx/raw?ref=master" >> onem2m_delimiter_start.docx curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/pipeline%2Fonem2m_delimiter_start%2Edocx/raw?ref=master" >> onem2m_delimiter_start.docx
- | - |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_end%2Edocx/raw?ref=master" >> onem2m_delimiter_end.docx curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/pipeline%2Fonem2m_delimiter_end%2Edocx/raw?ref=master" >> onem2m_delimiter_end.docx
- | - |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_coversheet_template%2Edocx/raw?ref=master" >> onem2m_coversheet_template.docx curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/pipeline%2Fonem2m_coversheet_template%2Edocx/raw?ref=master" >> onem2m_coversheet_template.docx
- | - |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/pipeline%2FSpec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx
script: script:
- echo 'CR contribution' - echo 'CR contribution'
- mkdir docs - mkdir docs
...@@ -89,10 +89,10 @@ Baseline contribution: ...@@ -89,10 +89,10 @@ Baseline contribution:
when: on_success when: on_success
before_script: before_script:
- | - |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_spec%2Esh/raw?ref=master" >> publish_spec.sh curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/pipeline%2Fpublish_spec%2Esh/raw?ref=master" >> publish_spec.sh
- chmod +x publish_spec.sh - chmod +x publish_spec.sh
- | - |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/pipeline%2FSpec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx
- | - |
export SPEC_NAME=$(ls | grep T*.md | cut -d'.' -f1) export SPEC_NAME=$(ls | grep T*.md | cut -d'.' -f1)
export BASELINE_VERSION=$(echo ${CI_MERGE_REQUEST_TITLE} | grep -o v.*_baseline | cut -d'_' -f1-3 ) export BASELINE_VERSION=$(echo ${CI_MERGE_REQUEST_TITLE} | grep -o v.*_baseline | cut -d'_' -f1-3 )
...@@ -124,10 +124,10 @@ Publish spec: ...@@ -124,10 +124,10 @@ Publish spec:
- tags - tags
before_script: before_script:
- | - |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_spec%2Esh/raw?ref=master" >> publish_spec.sh curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/pipeline%2Fpublish_spec%2Esh/raw?ref=master" >> publish_spec.sh
- chmod +x publish_spec.sh - chmod +x publish_spec.sh
- | - |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/pipeline%2FSpec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx
- | - |
export SPEC_NAME=$(ls | grep -E "(TS|TR|WI).*\.md" | cut -d'.' -f1) export SPEC_NAME=$(ls | grep -E "(TS|TR|WI).*\.md" | cut -d'.' -f1)
script: script:
...@@ -142,7 +142,7 @@ Publish spec: ...@@ -142,7 +142,7 @@ Publish spec:
pages: pages:
stage: web stage: web
image: python:3.9.18-slim-bullseye image: forge.etsi.org:5050/cti/tools/generatespecwebsite:restructure
tags: tags:
- docker - docker
#needs: ["Publish spec"] Uncomment when finishing testing #needs: ["Publish spec"] Uncomment when finishing testing
...@@ -184,40 +184,40 @@ pages: ...@@ -184,40 +184,40 @@ pages:
- | - |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2Fonem2m_sq%2Epng/raw?ref=master" >> onem2m_sq.png curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2Fonem2m_sq%2Epng/raw?ref=master" >> onem2m_sq.png
- mv onem2m_sq.png docs/images/ - mv onem2m_sq.png docs/images/
- | #- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2FWord%2Epng/raw?ref=master" >> Word.png # curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2FWord%2Epng/raw?ref=master" >> Word.png
- mkdir -p docs/download/images && mv Word.png docs/download/images/ - mkdir -p docs/download/images && mv generateSpecWebSite/images/Word.png docs/download/images/
- | #- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2FEpub%2Epng/raw?ref=master" >> Epub.png # curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2FEpub%2Epng/raw?ref=master" >> Epub.png
- mv Epub.png docs/download/images/ - mv generateSpecWebSite/images/Epub.png docs/download/images/
- | #- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2FPdf%2Epng/raw?ref=master" >> Pdf.png # curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2FPdf%2Epng/raw?ref=master" >> Pdf.png
- mv Pdf.png docs/download/images/ - mv generateSpecWebSite/images/ Pdf.png docs/download/images/
- | #- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fstylesheets%2Fextra%2Ecss/raw?ref=master" >> extra.css # curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fstylesheets%2Fextra%2Ecss/raw?ref=master" >> extra.css
- mkdir -p docs/stylesheets && mv extra.css docs/stylesheets/ - mkdir -p docs/stylesheets && mv generateSpecWebSite/stylesheets/extra.css docs/stylesheets/
- | #- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fmkdocs%2Eyml/raw?ref=master" >> mkdocs.yml # curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fmkdocs%2Eyml/raw?ref=master" >> mkdocs.yml
- | #- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2FindexDownload%2Emd/raw?ref=master" >> indexDownload.md # curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2FindexDownload%2Emd/raw?ref=master" >> indexDownload.md
- mkdir -p docs/download && mv indexDownload.md docs/download/index.md - mkdir -p docs/download && mv generateSpecWebSite/indexDownload.md docs/download/index.md
- | #- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2FtoMkdocs%2Epy/raw?ref=master" >> toMkdocs.py # curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2FtoMkdocs%2Epy/raw?ref=master" >> toMkdocs.py
- | #- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2FmarkdownTools%2Epy/raw?ref=master" >> markdownTools.py # curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2FmarkdownTools%2Epy/raw?ref=master" >> markdownTools.py
- | #- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2FgridTableTools%2Epy/raw?ref=master" >> gridTableTools.py # curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2FgridTableTools%2Epy/raw?ref=master" >> gridTableTools.py
- | #- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2FregexMatches%2Epy/raw?ref=master" >> regexMatches.py # curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2FregexMatches%2Epy/raw?ref=master" >> regexMatches.py
- | - |
export SPEC_NAME=$(ls | grep -E "(TS|TR|WI).*\.md" | cut -d'.' -f1) export SPEC_NAME=$(ls | grep -E "(TS|TR|WI).*\.md" | cut -d'.' -f1)
- | #- |
curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/spec_on_pages%2Esh/raw?ref=master" >> spec_on_pages.sh # curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/spec_on_pages%2Esh/raw?ref=master" >> spec_on_pages.sh
- chmod +x spec_on_pages.sh #- chmod +x spec_on_pages.sh
- echo "'$TAG_NAME'" - echo "'$TAG_NAME'"
script: script:
- ./spec_on_pages.sh toMkdocs.py $MIKE_ACCESS_TOKEN $CI_SERVER_HOST $CI_PROJECT_PATH "$GITLAB_USER_NAME" $GITLAB_USER_EMAIL $CI_COMMIT_SHA $SPEC_NAME $CI_PROJECT_NAME $TAG_NAME $CLEAN_WEB_PAGES - ./generateSpecWebSite/spec_on_pages.sh generateSpecWebSite/toMkdocs.py $MIKE_ACCESS_TOKEN $CI_SERVER_HOST $CI_PROJECT_PATH "$GITLAB_USER_NAME" $GITLAB_USER_EMAIL $CI_COMMIT_SHA $SPEC_NAME $CI_PROJECT_NAME $TAG_NAME $CLEAN_WEB_PAGES
artifacts: artifacts:
paths: paths:
- public/ - public/
\ No newline at end of file
File moved
...@@ -10,7 +10,7 @@ ...@@ -10,7 +10,7 @@
#!/bin/bash #!/bin/bash
FORGELIB_DOCKER_IMAGE=forge.3gpp.org:5050/tools/3gpp-scripts/forgelib:v2.22.0 FORGELIB_DOCKER_IMAGE=forge.3gpp.org:5050/tools/3gpp-scripts/forgelib:v2.22.0
GENERATE_CHANGEMARKS_DOCKER_IMAGE=generatechangemarks:master GENERATE_CHANGEMARKS_DOCKER_IMAGE=forge.etsi.org:5050/cti/tools/generatecr:restructure
DOCKER_IMAGE=pandoc/core:3.1.1.0 DOCKER_IMAGE=pandoc/core:3.1.1.0
echo "\n------ Checking for docker image --------" echo "\n------ Checking for docker image --------"
......
File moved
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment