From 89e36bd1b8555d84d0afdab3d41b1ad050d6acba Mon Sep 17 00:00:00 2001
From: Miguel Angel Reina Ortega <miguelangel.reinaortega@etsi.org>
Date: Wed, 13 Dec 2023 17:32:43 +0100
Subject: [PATCH] Add functionality to upgrade index.html in all projects

---
 .gitlab-ci.yml                     |  97 +++----------------------
 generateChangemarks/.gitlab-ci.yml | 109 +++++++++++++++++++++++++++++
 publish_on_pages.sh                |   9 ++-
 updateIndex.py                     |  54 +++++++++++++-
 upgrade_pages.sh                   |  30 ++++++++
 5 files changed, 207 insertions(+), 92 deletions(-)
 create mode 100644 generateChangemarks/.gitlab-ci.yml
 create mode 100755 upgrade_pages.sh

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 570d00f..79e90c2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,21 +6,14 @@
 
 workflow:
   rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-    - if: $CI_COMMIT_TAG
-    - if: $CI_COMMIT_BRANCH && $CI_PROJECT_NAME == "scripts"
-    - if: $CI_PIPELINE_SOURCE == "web"
-    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
-      when: never
+    - if: $CI_COMMIT_BRANCH
 
 variables:
 
 stages:
   - build
-  - generation
-  - publication
-  - web
-    
+  - upgrade
+
 Build pythonForPandocFilter docker image:
   stage: build
   before_script: cd generateChangemarks
@@ -35,85 +28,11 @@ Build pythonForPandocFilter docker image:
         - generateChangemarks/changemarks.py
         - generateChangemarks/addTrackedChanges.py
         
-Word CR text:
-  stage: generation
-  only:
-    - merge_requests
+Upgrade index for pages:
+  stage: upgrade
   before_script:
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/generate_changemarks%2Esh/raw?ref=master" >> generate_changemarks.sh
-    - chmod +x generate_changemarks.sh
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_start%2Edocx/raw?ref=master" >> onem2m_delimiter_start.docx
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_end%2Edocx/raw?ref=master" >> onem2m_delimiter_end.docx
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_coversheet_template%2Edocx/raw?ref=master" >> onem2m_coversheet_template.docx
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx
-  script:
-    - echo 'CR contribution'
-    - mkdir docs
-    - ./generate_changemarks.sh ${CI_PROJECT_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${CI_MERGE_REQUEST_IID} onem2m_delimiter_start.docx onem2m_delimiter_end.docx onem2m_coversheet_template.docx onem2m_spec_template.docx
-  artifacts:
-    paths:
-      - docs/
-    expose_as: 'Word CR text'
-
-Protect branch:
-  stage: generation
-  when: on_success
-  needs: ["Word CR text"]
-  only:
-    - merge_requests
   script:
-    - |
-     curl --request POST --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/protected_branches?name=${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}&merge_access_level=30"
-
-
-Publish spec:
-  stage: publication
-  only:
-    - tags
-  before_script:
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_spec%2Esh/raw?ref=master" >> publish_spec.sh
-    - chmod +x publish_spec.sh
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx
-    - |
-     export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1)
-  script:
-    - echo 'Publishing spec'
-    - ./publish_spec.sh ${CI_PROJECT_URL} ${CI_COMMIT_TAG} onem2m_spec_template.docx $SPEC_NAME
-  artifacts:
-    name: "${SPEC_NAME}_${CI_COMMIT_TAG}"
-    paths:
-      - "*_${CI_COMMIT_TAG}.docx"
-
-pages:
-  stage: web
-  when: on_success
+    - ./upgrade_pages.sh ${ACCESS_TOKEN}
   rules:
-    - if: ($CLEAN_WEB_PAGES == 'false' || $CLEAN_WEB_PAGES == 'true') && $CI_COMMIT_TAG
-      variables:
-        TAG_NAME: $CI_COMMIT_TAG
-    - if: ($CLEAN_WEB_PAGES != "false" && $CLEAN_WEB_PAGES != "true") && $CI_PIPELINE_SOURCE == "web"
-      variables:
-        TAG_NAME: $CLEAN_WEB_PAGES
-  before_script:
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_on_pages%2Esh/raw?ref=master" >> publish_on_pages.sh
-    - chmod +x publish_on_pages.sh
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/updateIndex%2Epy/raw?ref=master" >> updateIndex.py
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/index%2Ehtml/raw?ref=master" >> index.html
-    - |
-     export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1)
-  script:
-    - echo 'Publishing on pages'
-    - ./publish_on_pages.sh updateIndex.py $CI_PAGES_URL ${CI_PROJECT_NAME} $TAG_NAME index.html $CLEAN_WEB_PAGES $SPEC_NAME
-  artifacts:
-    paths:
-      - public
\ No newline at end of file
+    - changes:
+        - index.html
\ No newline at end of file
diff --git a/generateChangemarks/.gitlab-ci.yml b/generateChangemarks/.gitlab-ci.yml
new file mode 100644
index 0000000..d5f6a33
--- /dev/null
+++ b/generateChangemarks/.gitlab-ci.yml
@@ -0,0 +1,109 @@
+# CI/CD:
+#
+# GENERATION triggered by:
+#  - merge request
+#
+
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Creation of Merge Request (generation of CR)
+    - if: $CI_COMMIT_TAG # Creation of tag (publish spec)
+    - if: $CI_PIPELINE_SOURCE == "web" # Removal of a specific entry
+    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
+      when: never # Pipeline to be run manually as defined in the contribution procedure as it is not known when the CR is ready
+    - if: $CI_PIPELINE_SOURCE == "trigger" # Upgrade of the index.html page
+
+variables:
+
+stages:
+  - generation
+  - publication
+  - web
+
+Word CR:
+  stage: generation
+  only:
+    - merge_requests
+  before_script:
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/generate_changemarks%2Esh/raw?ref=master" >> generate_changemarks.sh
+    - chmod +x generate_changemarks.sh
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_start%2Edocx/raw?ref=master" >> onem2m_delimiter_start.docx
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_end%2Edocx/raw?ref=master" >> onem2m_delimiter_end.docx
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_coversheet_template%2Edocx/raw?ref=master" >> onem2m_coversheet_template.docx
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx
+  script:
+    - echo 'CR contribution'
+    - mkdir docs
+    - ./generate_changemarks.sh ${CI_PROJECT_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${CI_MERGE_REQUEST_IID} onem2m_delimiter_start.docx onem2m_delimiter_end.docx onem2m_coversheet_template.docx onem2m_spec_template.docx
+  artifacts:
+    paths:
+      - docs/
+    expose_as: 'Word CR'
+
+Protect branch:
+  stage: generation
+  when: on_success
+  needs: ["Word CR"]
+  only:
+    - merge_requests
+  script:
+    - |
+     curl --request POST --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/protected_branches?name=${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}&merge_access_level=30"
+
+
+Publish spec:
+  stage: publication
+  only:
+    - tags
+  before_script:
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_spec%2Esh/raw?ref=master" >> publish_spec.sh
+    - chmod +x publish_spec.sh
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx
+    - |
+     export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1)
+  script:
+    - echo 'Publishing spec'
+    - ./publish_spec.sh ${CI_PROJECT_URL} ${CI_COMMIT_TAG} onem2m_spec_template.docx $SPEC_NAME
+  artifacts:
+    name: "${SPEC_NAME}_${CI_COMMIT_TAG}"
+    paths:
+      - "*_${CI_COMMIT_TAG}.docx"
+
+pages:
+  stage: web
+  when: on_success
+  rules:
+    - if: ($CLEAN_WEB_PAGES == 'false' || $CLEAN_WEB_PAGES == 'true') && $CI_COMMIT_TAG
+      variables:
+        TAG_NAME: $CI_COMMIT_TAG
+    - if: ($CLEAN_WEB_PAGES != "false" && $CLEAN_WEB_PAGES != "true") && $CI_PIPELINE_SOURCE == "web"
+      variables:
+        TAG_NAME: $CLEAN_WEB_PAGES
+    - if: $CI_PIPELINE_SOURCE == "trigger"
+      variables:
+        TAG_NAME: "upgrade"
+  before_script:
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_on_pages%2Esh/raw?ref=master" >> publish_on_pages.sh
+    - chmod +x publish_on_pages.sh
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/updateIndex%2Epy/raw?ref=master" >> updateIndex.py
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/index%2Ehtml/raw?ref=master" >> index.html
+    - |
+     export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1)
+    - |
+     sed -i 's/PROJECT/'${CI_PROJECT_NAME^^}'/g' index.html
+  script:
+    - echo 'Publishing on pages'
+    -  ./publish_on_pages.sh updateIndex.py $CI_PAGES_URL ${CI_PROJECT_NAME} $TAG_NAME index.html $CLEAN_WEB_PAGES $SPEC_NAME;
+  artifacts:
+    paths:
+      - public
\ No newline at end of file
diff --git a/publish_on_pages.sh b/publish_on_pages.sh
index c86153f..38cc673 100644
--- a/publish_on_pages.sh
+++ b/publish_on_pages.sh
@@ -5,7 +5,7 @@ DOCKER_IMAGE=python:3.9-slim-bullseye
 echo "\n------ Checking for docker image --------"
 docker pull "$DOCKER_IMAGE"
 echo "Pages URL:" $2
-echo "PROJECT NAME:" $3
+echo "PROJECT NAME:" $3 # Not used anymore, consider to remove it
 echo "TAG NAME:" $4
 
 echo "------ Get the previous GitLab Pages content --------"
@@ -15,7 +15,10 @@ if [ -e "content.zip" ]; then unzip "content.zip"; rm "content.zip"; fi
 set -e
 
 echo "------ Add/update content --------"
-if [ $6 == 'true' ]; then 
+if [ $4 == 'upgrade' ]; then
+    echo 'Upgrading index.html...'
+    docker run --rm -v $(pwd):/tmp -w /tmp "$DOCKER_IMAGE" pip install bs4 && python3 $1 -u "$7_$4" "$4/$7_$4.docx" "$5"
+elif [ $6 == 'true' ]; then
     echo 'Removing all web pages content...'
 	rm -r public/*;
 elif [ $6 == 'false' ]; then
@@ -41,3 +44,5 @@ exit 0
 #$CI_PROJECT_NAME
 #$CI_COMMIT_REF_NAME
 #index.html
+#CLEAN_WEB_PAGES
+#SPEC_NAME
diff --git a/updateIndex.py b/updateIndex.py
index 516ea0b..9064977 100644
--- a/updateIndex.py
+++ b/updateIndex.py
@@ -9,6 +9,53 @@
 from bs4 import BeautifulSoup
 import argparse, os
 
+
+def upgradeIndex(args: argparse.Namespace) -> None:
+    # Check if index.htm exists
+    if not os.path.exists('public/index.html'):
+        print("'index.html' does not exist yet, Upgrade is not needed")
+        exit()
+    else:
+        # Read the source HTML file
+        with open('public/index.html', 'r', encoding='utf-8') as source_file:
+            source_html = source_file.read()
+
+        # Read the destination HTML file
+        with open(args.indexTemplate, 'r', encoding='utf-8') as dest_file:
+            dest_html = dest_file.read()
+
+        print(source_html)
+        print(dest_html)
+        # Parse both HTML files
+        source_soup = BeautifulSoup(source_html, 'html.parser')
+        dest_soup = BeautifulSoup(dest_html, 'html.parser')
+
+        print(source_soup)
+        print(dest_soup)
+        # Find the <ul> element in both the source and destination HTML
+        source_ul = source_soup.find('ul', id='document-list')
+        dest_ul = dest_soup.find('ul', id='document-list')
+
+        print(source_ul)
+        print(dest_ul)
+        # Copy the contents of the source <ul> to the destination <ul>
+        if dest_ul and source_ul:
+            dest_ul.contents.extend(source_ul.contents)
+        else:
+            print("ERROR: List not found in source and destination index files...")
+            exit(1)
+
+        # Save the updated destination HTML
+        with open('public/index.html', 'w', encoding='utf-8') as dest_updated_file:
+            dest_updated_file.write(str(dest_soup))
+
+
+        # Read index HTML
+        with open('public/index.html', 'r', encoding='utf-8') as file:
+            html_content = file.read()
+        print("Index.html upgraded")
+
+
 def updateIndex(args:argparse.Namespace) -> None:
     
     # Check if index.htm exists
@@ -63,10 +110,15 @@ if __name__ == '__main__':
     parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument('-re', '--removeEntry', action='store_true', required=False, default=False,
                         help="Indicate to remove the given entry")
+    parser.add_argument('-u', '--upgrade', action='store_true', required=False, default=False,
+                        help="Upgrade the index.html")
     parser.add_argument('documentName', help = 'Document name to add to the list')
     parser.add_argument('documentLink', help = 'Document location to add to the list')
     parser.add_argument('indexTemplate', help = 'Index template document')
     args = parser.parse_args()
 
-    updateIndex(args)
+    if args.upgrade:
+        upgradeIndex(args)
+    else:
+        updateIndex(args)
 
diff --git a/upgrade_pages.sh b/upgrade_pages.sh
new file mode 100755
index 0000000..d29cc07
--- /dev/null
+++ b/upgrade_pages.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+GITLAB_HOST="http://git.onem2m.org"
+SPECIFICATIONS_GROUP="2501"
+NS_GITLAB_HOST=$(echo $GITLAB_HOST | cut -d'/' -f3)
+
+echo "\n------ Getting all projects IDs --------"
+# Get a list of projects in the group using GitLab API
+#PROJECT_IDS=$(curl --header "PRIVATE-TOKEN: $1" "$GITLAB_HOST/api/v4/groups/$SPECIFICATIONS_GROUP/projects" | grep -oP '"id":(?!'$SPECIFICATIONS_GROUP')\d+' | cut -d':' -f2)
+PROJECT_IDS=$(curl "$GITLAB_HOST/api/v4/groups/$SPECIFICATIONS_GROUP/projects" | grep -oP '"id":(?!'$SPECIFICATIONS_GROUP')\d+' | cut -d':' -f2)
+echo $PROJECT_IDS
+
+#PROJECT_IDS="41" # Use TS-0019 for debugging
+# Trigger pipeline for each project. Triggered pipeline pages job will handle the upgrade of the index.html
+for PROJECT_ID in $PROJECT_IDS; do
+	echo "------ Preparing specification project --------"
+	#WORKAROUND UNTIL ACCESS TOKEN WORKS FOR TRIGGERING A PIPELINE (right now, it is allowed but it seems NOT to work)
+	#create a pipeline trigger token first
+	echo "------ Creating pipeline trigger token --------"
+	PIPELINE_TRIGGER_TOKEN_ID=$(curl --request POST --header "PRIVATE-TOKEN:$1" --form description="Temporary pipeline trigger token for tools/scripts" https://git.onem2m.org/api/v4/projects/$PROJECT_ID/triggers | grep -o '"id":[^,]*,"token"'|cut -d',' -f1 | cut -d':' -f2)
+    PIPELINE_TRIGGER_TOKEN=$(curl --header "PRIVATE-TOKEN:$1" https://git.onem2m.org/api/v4/projects/$PROJECT_ID/triggers/$PIPELINE_TRIGGER_TOKEN_ID | grep -o '"token":[^,]*'|cut -d'"' -f4)
+    echo "------ Triggering pipeline --------"
+	#trigger the pipeline
+    curl -X POST --fail -F token=$PIPELINE_TRIGGER_TOKEN -F ref=main https://git.onem2m.org/api/v4/projects/$PROJECT_ID/trigger/pipeline
+    echo "------ Deleting pipeline trigger token (NOTE that information about the token that caused the pipeline will NOT be available for the pipeline) --------"
+	#remove the pipeline trigger token
+    curl --request DELETE --header "PRIVATE-TOKEN:$1" "https://git.onem2m.org/api/v4/projects/$PROJECT_ID/triggers/$PIPELINE_TRIGGER_TOKEN_ID"
+done
+
+exit 0
\ No newline at end of file
-- 
GitLab