From 89e36bd1b8555d84d0afdab3d41b1ad050d6acba Mon Sep 17 00:00:00 2001 From: Miguel Angel Reina Ortega <miguelangel.reinaortega@etsi.org> Date: Wed, 13 Dec 2023 17:32:43 +0100 Subject: [PATCH] Add functionality to upgrade index.html in all projects --- .gitlab-ci.yml | 97 +++---------------------- generateChangemarks/.gitlab-ci.yml | 109 +++++++++++++++++++++++++++++ publish_on_pages.sh | 9 ++- updateIndex.py | 54 +++++++++++++- upgrade_pages.sh | 30 ++++++++ 5 files changed, 207 insertions(+), 92 deletions(-) create mode 100644 generateChangemarks/.gitlab-ci.yml create mode 100755 upgrade_pages.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 570d00f..79e90c2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,21 +6,14 @@ workflow: rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: $CI_COMMIT_TAG - - if: $CI_COMMIT_BRANCH && $CI_PROJECT_NAME == "scripts" - - if: $CI_PIPELINE_SOURCE == "web" - - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS - when: never + - if: $CI_COMMIT_BRANCH variables: stages: - build - - generation - - publication - - web - + - upgrade + Build pythonForPandocFilter docker image: stage: build before_script: cd generateChangemarks @@ -35,85 +28,11 @@ Build pythonForPandocFilter docker image: - generateChangemarks/changemarks.py - generateChangemarks/addTrackedChanges.py -Word CR text: - stage: generation - only: - - merge_requests +Upgrade index for pages: + stage: upgrade before_script: - - | - curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/generate_changemarks%2Esh/raw?ref=master" >> generate_changemarks.sh - - chmod +x generate_changemarks.sh - - | - curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_start%2Edocx/raw?ref=master" >> onem2m_delimiter_start.docx - - | - curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_end%2Edocx/raw?ref=master" >> onem2m_delimiter_end.docx - - | - curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_coversheet_template%2Edocx/raw?ref=master" >> onem2m_coversheet_template.docx - - | - curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx - script: - - echo 'CR contribution' - - mkdir docs - - ./generate_changemarks.sh ${CI_PROJECT_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${CI_MERGE_REQUEST_IID} onem2m_delimiter_start.docx onem2m_delimiter_end.docx onem2m_coversheet_template.docx onem2m_spec_template.docx - artifacts: - paths: - - docs/ - expose_as: 'Word CR text' - -Protect branch: - stage: generation - when: on_success - needs: ["Word CR text"] - only: - - merge_requests script: - - | - curl --request POST --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/protected_branches?name=${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}&merge_access_level=30" - - -Publish spec: - stage: publication - only: - - tags - before_script: - - | - curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_spec%2Esh/raw?ref=master" >> publish_spec.sh - - chmod +x publish_spec.sh - - | - curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx - - | - export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1) - script: - - echo 'Publishing spec' - - ./publish_spec.sh ${CI_PROJECT_URL} ${CI_COMMIT_TAG} onem2m_spec_template.docx $SPEC_NAME - artifacts: - name: "${SPEC_NAME}_${CI_COMMIT_TAG}" - paths: - - "*_${CI_COMMIT_TAG}.docx" - -pages: - stage: web - when: on_success + - ./upgrade_pages.sh ${ACCESS_TOKEN} rules: - - if: ($CLEAN_WEB_PAGES == 'false' || $CLEAN_WEB_PAGES == 'true') && $CI_COMMIT_TAG - variables: - TAG_NAME: $CI_COMMIT_TAG - - if: ($CLEAN_WEB_PAGES != "false" && $CLEAN_WEB_PAGES != "true") && $CI_PIPELINE_SOURCE == "web" - variables: - TAG_NAME: $CLEAN_WEB_PAGES - before_script: - - | - curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_on_pages%2Esh/raw?ref=master" >> publish_on_pages.sh - - chmod +x publish_on_pages.sh - - | - curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/updateIndex%2Epy/raw?ref=master" >> updateIndex.py - - | - curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/index%2Ehtml/raw?ref=master" >> index.html - - | - export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1) - script: - - echo 'Publishing on pages' - - ./publish_on_pages.sh updateIndex.py $CI_PAGES_URL ${CI_PROJECT_NAME} $TAG_NAME index.html $CLEAN_WEB_PAGES $SPEC_NAME - artifacts: - paths: - - public \ No newline at end of file + - changes: + - index.html \ No newline at end of file diff --git a/generateChangemarks/.gitlab-ci.yml b/generateChangemarks/.gitlab-ci.yml new file mode 100644 index 0000000..d5f6a33 --- /dev/null +++ b/generateChangemarks/.gitlab-ci.yml @@ -0,0 +1,109 @@ +# CI/CD: +# +# GENERATION triggered by: +# - merge request +# + +workflow: + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Creation of Merge Request (generation of CR) + - if: $CI_COMMIT_TAG # Creation of tag (publish spec) + - if: $CI_PIPELINE_SOURCE == "web" # Removal of a specific entry + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS + when: never # Pipeline to be run manually as defined in the contribution procedure as it is not known when the CR is ready + - if: $CI_PIPELINE_SOURCE == "trigger" # Upgrade of the index.html page + +variables: + +stages: + - generation + - publication + - web + +Word CR: + stage: generation + only: + - merge_requests + before_script: + - | + curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/generate_changemarks%2Esh/raw?ref=master" >> generate_changemarks.sh + - chmod +x generate_changemarks.sh + - | + curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_start%2Edocx/raw?ref=master" >> onem2m_delimiter_start.docx + - | + curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_end%2Edocx/raw?ref=master" >> onem2m_delimiter_end.docx + - | + curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_coversheet_template%2Edocx/raw?ref=master" >> onem2m_coversheet_template.docx + - | + curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx + script: + - echo 'CR contribution' + - mkdir docs + - ./generate_changemarks.sh ${CI_PROJECT_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${CI_MERGE_REQUEST_IID} onem2m_delimiter_start.docx onem2m_delimiter_end.docx onem2m_coversheet_template.docx onem2m_spec_template.docx + artifacts: + paths: + - docs/ + expose_as: 'Word CR' + +Protect branch: + stage: generation + when: on_success + needs: ["Word CR"] + only: + - merge_requests + script: + - | + curl --request POST --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/protected_branches?name=${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}&merge_access_level=30" + + +Publish spec: + stage: publication + only: + - tags + before_script: + - | + curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_spec%2Esh/raw?ref=master" >> publish_spec.sh + - chmod +x publish_spec.sh + - | + curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx + - | + export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1) + script: + - echo 'Publishing spec' + - ./publish_spec.sh ${CI_PROJECT_URL} ${CI_COMMIT_TAG} onem2m_spec_template.docx $SPEC_NAME + artifacts: + name: "${SPEC_NAME}_${CI_COMMIT_TAG}" + paths: + - "*_${CI_COMMIT_TAG}.docx" + +pages: + stage: web + when: on_success + rules: + - if: ($CLEAN_WEB_PAGES == 'false' || $CLEAN_WEB_PAGES == 'true') && $CI_COMMIT_TAG + variables: + TAG_NAME: $CI_COMMIT_TAG + - if: ($CLEAN_WEB_PAGES != "false" && $CLEAN_WEB_PAGES != "true") && $CI_PIPELINE_SOURCE == "web" + variables: + TAG_NAME: $CLEAN_WEB_PAGES + - if: $CI_PIPELINE_SOURCE == "trigger" + variables: + TAG_NAME: "upgrade" + before_script: + - | + curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_on_pages%2Esh/raw?ref=master" >> publish_on_pages.sh + - chmod +x publish_on_pages.sh + - | + curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/updateIndex%2Epy/raw?ref=master" >> updateIndex.py + - | + curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/index%2Ehtml/raw?ref=master" >> index.html + - | + export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1) + - | + sed -i 's/PROJECT/'${CI_PROJECT_NAME^^}'/g' index.html + script: + - echo 'Publishing on pages' + - ./publish_on_pages.sh updateIndex.py $CI_PAGES_URL ${CI_PROJECT_NAME} $TAG_NAME index.html $CLEAN_WEB_PAGES $SPEC_NAME; + artifacts: + paths: + - public \ No newline at end of file diff --git a/publish_on_pages.sh b/publish_on_pages.sh index c86153f..38cc673 100644 --- a/publish_on_pages.sh +++ b/publish_on_pages.sh @@ -5,7 +5,7 @@ DOCKER_IMAGE=python:3.9-slim-bullseye echo "\n------ Checking for docker image --------" docker pull "$DOCKER_IMAGE" echo "Pages URL:" $2 -echo "PROJECT NAME:" $3 +echo "PROJECT NAME:" $3 # Not used anymore, consider to remove it echo "TAG NAME:" $4 echo "------ Get the previous GitLab Pages content --------" @@ -15,7 +15,10 @@ if [ -e "content.zip" ]; then unzip "content.zip"; rm "content.zip"; fi set -e echo "------ Add/update content --------" -if [ $6 == 'true' ]; then +if [ $4 == 'upgrade' ]; then + echo 'Upgrading index.html...' + docker run --rm -v $(pwd):/tmp -w /tmp "$DOCKER_IMAGE" pip install bs4 && python3 $1 -u "$7_$4" "$4/$7_$4.docx" "$5" +elif [ $6 == 'true' ]; then echo 'Removing all web pages content...' rm -r public/*; elif [ $6 == 'false' ]; then @@ -41,3 +44,5 @@ exit 0 #$CI_PROJECT_NAME #$CI_COMMIT_REF_NAME #index.html +#CLEAN_WEB_PAGES +#SPEC_NAME diff --git a/updateIndex.py b/updateIndex.py index 516ea0b..9064977 100644 --- a/updateIndex.py +++ b/updateIndex.py @@ -9,6 +9,53 @@ from bs4 import BeautifulSoup import argparse, os + +def upgradeIndex(args: argparse.Namespace) -> None: + # Check if index.htm exists + if not os.path.exists('public/index.html'): + print("'index.html' does not exist yet, Upgrade is not needed") + exit() + else: + # Read the source HTML file + with open('public/index.html', 'r', encoding='utf-8') as source_file: + source_html = source_file.read() + + # Read the destination HTML file + with open(args.indexTemplate, 'r', encoding='utf-8') as dest_file: + dest_html = dest_file.read() + + print(source_html) + print(dest_html) + # Parse both HTML files + source_soup = BeautifulSoup(source_html, 'html.parser') + dest_soup = BeautifulSoup(dest_html, 'html.parser') + + print(source_soup) + print(dest_soup) + # Find the <ul> element in both the source and destination HTML + source_ul = source_soup.find('ul', id='document-list') + dest_ul = dest_soup.find('ul', id='document-list') + + print(source_ul) + print(dest_ul) + # Copy the contents of the source <ul> to the destination <ul> + if dest_ul and source_ul: + dest_ul.contents.extend(source_ul.contents) + else: + print("ERROR: List not found in source and destination index files...") + exit(1) + + # Save the updated destination HTML + with open('public/index.html', 'w', encoding='utf-8') as dest_updated_file: + dest_updated_file.write(str(dest_soup)) + + + # Read index HTML + with open('public/index.html', 'r', encoding='utf-8') as file: + html_content = file.read() + print("Index.html upgraded") + + def updateIndex(args:argparse.Namespace) -> None: # Check if index.htm exists @@ -63,10 +110,15 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-re', '--removeEntry', action='store_true', required=False, default=False, help="Indicate to remove the given entry") + parser.add_argument('-u', '--upgrade', action='store_true', required=False, default=False, + help="Upgrade the index.html") parser.add_argument('documentName', help = 'Document name to add to the list') parser.add_argument('documentLink', help = 'Document location to add to the list') parser.add_argument('indexTemplate', help = 'Index template document') args = parser.parse_args() - updateIndex(args) + if args.upgrade: + upgradeIndex(args) + else: + updateIndex(args) diff --git a/upgrade_pages.sh b/upgrade_pages.sh new file mode 100755 index 0000000..d29cc07 --- /dev/null +++ b/upgrade_pages.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +GITLAB_HOST="http://git.onem2m.org" +SPECIFICATIONS_GROUP="2501" +NS_GITLAB_HOST=$(echo $GITLAB_HOST | cut -d'/' -f3) + +echo "\n------ Getting all projects IDs --------" +# Get a list of projects in the group using GitLab API +#PROJECT_IDS=$(curl --header "PRIVATE-TOKEN: $1" "$GITLAB_HOST/api/v4/groups/$SPECIFICATIONS_GROUP/projects" | grep -oP '"id":(?!'$SPECIFICATIONS_GROUP')\d+' | cut -d':' -f2) +PROJECT_IDS=$(curl "$GITLAB_HOST/api/v4/groups/$SPECIFICATIONS_GROUP/projects" | grep -oP '"id":(?!'$SPECIFICATIONS_GROUP')\d+' | cut -d':' -f2) +echo $PROJECT_IDS + +#PROJECT_IDS="41" # Use TS-0019 for debugging +# Trigger pipeline for each project. Triggered pipeline pages job will handle the upgrade of the index.html +for PROJECT_ID in $PROJECT_IDS; do + echo "------ Preparing specification project --------" + #WORKAROUND UNTIL ACCESS TOKEN WORKS FOR TRIGGERING A PIPELINE (right now, it is allowed but it seems NOT to work) + #create a pipeline trigger token first + echo "------ Creating pipeline trigger token --------" + PIPELINE_TRIGGER_TOKEN_ID=$(curl --request POST --header "PRIVATE-TOKEN:$1" --form description="Temporary pipeline trigger token for tools/scripts" https://git.onem2m.org/api/v4/projects/$PROJECT_ID/triggers | grep -o '"id":[^,]*,"token"'|cut -d',' -f1 | cut -d':' -f2) + PIPELINE_TRIGGER_TOKEN=$(curl --header "PRIVATE-TOKEN:$1" https://git.onem2m.org/api/v4/projects/$PROJECT_ID/triggers/$PIPELINE_TRIGGER_TOKEN_ID | grep -o '"token":[^,]*'|cut -d'"' -f4) + echo "------ Triggering pipeline --------" + #trigger the pipeline + curl -X POST --fail -F token=$PIPELINE_TRIGGER_TOKEN -F ref=main https://git.onem2m.org/api/v4/projects/$PROJECT_ID/trigger/pipeline + echo "------ Deleting pipeline trigger token (NOTE that information about the token that caused the pipeline will NOT be available for the pipeline) --------" + #remove the pipeline trigger token + curl --request DELETE --header "PRIVATE-TOKEN:$1" "https://git.onem2m.org/api/v4/projects/$PROJECT_ID/triggers/$PIPELINE_TRIGGER_TOKEN_ID" +done + +exit 0 \ No newline at end of file -- GitLab