Add functionality to upgrade index.html in all projects

fef655f6 · Miguel Angel Reina Ortega · 5ea48eaa · fef655f6 · fef655f6 · fef655f6
Commit fef655f6 authored 1 year ago by Miguel Angel Reina Ortega
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,20 +6,13 @@

 workflow:
  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-    - if: $CI_COMMIT_TAG
-    - if: $CI_COMMIT_BRANCH && $CI_PROJECT_NAME == "scripts"
-    - if: $CI_PIPELINE_SOURCE == "web"
-    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
-      when: never
+    - if: $CI_COMMIT_BRANCH

 variables:

 stages:
  - build
-  - generation
-  - publication
-  - web
+  - upgrade

 Build pythonForPandocFilter docker image:
  stage: build
@@ -35,85 +28,11 @@ Build pythonForPandocFilter docker image:
        - generateChangemarks/changemarks.py
        - generateChangemarks/addTrackedChanges.py
        
-Word CR text:
-  stage: generation
-  only:
-    - merge_requests
-  before_script:
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/generate_changemarks%2Esh/raw?ref=master" >> generate_changemarks.sh
-    - chmod +x generate_changemarks.sh
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_start%2Edocx/raw?ref=master" >> onem2m_delimiter_start.docx
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_end%2Edocx/raw?ref=master" >> onem2m_delimiter_end.docx
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_coversheet_template%2Edocx/raw?ref=master" >> onem2m_coversheet_template.docx
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx
-  script:
-    - echo 'CR contribution'
-    - mkdir docs
-    - ./generate_changemarks.sh ${CI_PROJECT_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${CI_MERGE_REQUEST_IID} onem2m_delimiter_start.docx onem2m_delimiter_end.docx onem2m_coversheet_template.docx onem2m_spec_template.docx
-  artifacts:
-    paths:
-      - docs/
-    expose_as: 'Word CR text'
-
-Protect branch:
-  stage: generation
-  when: on_success
-  needs: ["Word CR text"]
-  only:
-    - merge_requests
-  script:
-    - |
-     curl --request POST --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/protected_branches?name=${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}&merge_access_level=30"
-
-
-Publish spec:
-  stage: publication
-  only:
-    - tags
+Upgrade index for pages:
+  stage: upgrade
  before_script:
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_spec%2Esh/raw?ref=master" >> publish_spec.sh
-    - chmod +x publish_spec.sh
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx
-    - |
-     export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1)
  script:
-    - echo 'Publishing spec'
-    - ./publish_spec.sh ${CI_PROJECT_URL} ${CI_COMMIT_TAG} onem2m_spec_template.docx $SPEC_NAME
-  artifacts:
-    name: "${SPEC_NAME}_${CI_COMMIT_TAG}"
-    paths:
-      - "*_${CI_COMMIT_TAG}.docx"
-
-pages:
-  stage: web
-  when: on_success
+    - ./upgrade_pages.sh ${ACCESS_TOKEN}
  rules:
-    - if: ($CLEAN_WEB_PAGES == 'false' || $CLEAN_WEB_PAGES == 'true') && $CI_COMMIT_TAG
-      variables:
-        TAG_NAME: $CI_COMMIT_TAG
-    - if: ($CLEAN_WEB_PAGES != "false" && $CLEAN_WEB_PAGES != "true") && $CI_PIPELINE_SOURCE == "web"
-      variables:
-        TAG_NAME: $CLEAN_WEB_PAGES
-  before_script:
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_on_pages%2Esh/raw?ref=master" >> publish_on_pages.sh
-    - chmod +x publish_on_pages.sh
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/updateIndex%2Epy/raw?ref=master" >> updateIndex.py
-    - |
-     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/index%2Ehtml/raw?ref=master" >> index.html
-    - |
-     export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1)
-  script:
-    - echo 'Publishing on pages'
-    - ./publish_on_pages.sh updateIndex.py $CI_PAGES_URL ${CI_PROJECT_NAME} $TAG_NAME index.html $CLEAN_WEB_PAGES $SPEC_NAME
-  artifacts:
-    paths:
-      - public
\ No newline at end of file
+    - changes:
+        - index.html
\ No newline at end of file
--- a/generateChangemarks/.gtilab-ci.yml
+++ b/generateChangemarks/.gtilab-ci.yml
+# CI/CD:
+#
+# GENERATION triggered by:
+#  - merge request
+#
+
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Creation of Merge Request (generation of CR)
+    - if: $CI_COMMIT_TAG # Creation of tag (publish spec)
+    - if: $CI_PIPELINE_SOURCE == "web" # Removal of a specific entry
+    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
+      when: never # Pipeline to be run manually as defined in the contribution procedure as it is not known when the CR is ready
+    - if: $CI_PIPELINE_SOURCE == "trigger" # Upgrade of the index.html page
+
+variables:
+
+stages:
+  - generation
+  - publication
+  - web
+
+Word CR:
+  stage: generation
+  only:
+    - merge_requests
+  before_script:
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/generate_changemarks%2Esh/raw?ref=master" >> generate_changemarks.sh
+    - chmod +x generate_changemarks.sh
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_start%2Edocx/raw?ref=master" >> onem2m_delimiter_start.docx
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_end%2Edocx/raw?ref=master" >> onem2m_delimiter_end.docx
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_coversheet_template%2Edocx/raw?ref=master" >> onem2m_coversheet_template.docx
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx
+  script:
+    - echo 'CR contribution'
+    - mkdir docs
+    - ./generate_changemarks.sh ${CI_PROJECT_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${CI_MERGE_REQUEST_IID} onem2m_delimiter_start.docx onem2m_delimiter_end.docx onem2m_coversheet_template.docx onem2m_spec_template.docx
+  artifacts:
+    paths:
+      - docs/
+    expose_as: 'Word CR'
+
+Protect branch:
+  stage: generation
+  when: on_success
+  needs: ["Word CR"]
+  only:
+    - merge_requests
+  script:
+    - |
+     curl --request POST --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/protected_branches?name=${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}&merge_access_level=30"
+
+
+Publish spec:
+  stage: publication
+  only:
+    - tags
+  before_script:
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_spec%2Esh/raw?ref=master" >> publish_spec.sh
+    - chmod +x publish_spec.sh
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx
+    - |
+     export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1)
+  script:
+    - echo 'Publishing spec'
+    - ./publish_spec.sh ${CI_PROJECT_URL} ${CI_COMMIT_TAG} onem2m_spec_template.docx $SPEC_NAME
+  artifacts:
+    name: "${SPEC_NAME}_${CI_COMMIT_TAG}"
+    paths:
+      - "*_${CI_COMMIT_TAG}.docx"
+
+pages:
+  stage: web
+  when: on_success
+  needs: ["Publish spec"]
+  rules:
+    - if: ($CLEAN_WEB_PAGES == 'false' || $CLEAN_WEB_PAGES == 'true') && $CI_COMMIT_TAG
+      variables:
+        TAG_NAME: $CI_COMMIT_TAG
+    - if: ($CLEAN_WEB_PAGES != "false" && $CLEAN_WEB_PAGES != "true") && $CI_PIPELINE_SOURCE == "web"
+      variables:
+        TAG_NAME: $CLEAN_WEB_PAGES
+    - if: $CI_PIPELINE_SOURCE == "trigger"
+      variables:
+        TAG_NAME: "upgrade"
+before_script:
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_on_pages%2Esh/raw?ref=master" >> publish_on_pages.sh
+    - chmod +x publish_on_pages.sh
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/updateIndex%2Epy/raw?ref=master" >> updateIndex.py
+    - |
+     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/index%2Ehtml/raw?ref=master" >> index.html
+    - |
+     export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1)
+  script:
+    - echo 'Publishing on pages'
+    -  ./publish_on_pages.sh updateIndex.py $CI_PAGES_URL ${CI_PROJECT_NAME} $TAG_NAME index.html $CLEAN_WEB_PAGES $SPEC_NAME;
+  artifacts:
+    paths:
+      - public
\ No newline at end of file
--- a/publish_on_pages.sh
+++ b/publish_on_pages.sh
@@ -5,7 +5,7 @@ DOCKER_IMAGE=python:3.9-slim-bullseye
 echo "\n------ Checking for docker image --------"
 docker pull "$DOCKER_IMAGE"
 echo "Pages URL:" $2
-echo "PROJECT NAME:" $3
+echo "PROJECT NAME:" $3 # Not used anymore, consider to remove it
 echo "TAG NAME:" $4

 echo "------ Get the previous GitLab Pages content --------"
@@ -15,7 +15,10 @@ if [ -e "content.zip" ]; then unzip "content.zip"; rm "content.zip"; fi
 set -e

 echo "------ Add/update content --------"
-if [ $6 == 'true' ]; then 
+if [ $4 == 'upgrade' ]; then
+    echo 'Upgrading index.html...'
+    docker run --rm -v $(pwd):/tmp -w /tmp "$DOCKER_IMAGE" pip install bs4 && python3 $1 -u "$7_$4" "$4/$7_$4.docx" "$5"
+elif [ $6 == 'true' ]; then
    echo 'Removing all web pages content...'
 	rm -r public/*;
 elif [ $6 == 'false' ]; then
@@ -41,3 +44,5 @@ exit 0
 #$CI_PROJECT_NAME
 #$CI_COMMIT_REF_NAME
 #index.html
+#CLEAN_WEB_PAGES
+#SPEC_NAME
--- a/updateIndex.py
+++ b/updateIndex.py
@@ -9,6 +9,43 @@
 from bs4 import BeautifulSoup
 import argparse, os

+
+def upgradeIndex(args: argparse.Namespace) -> None:
+    # Check if index.htm exists
+    if not os.path.exists('public/index.html'):
+        print("'index.html' does not exist yet, Upgrade is not needed")
+        exit()
+    else:
+        # Read the source HTML file
+        with open('public/index.html', 'r', encoding='utf-8') as source_file:
+            source_html = source_file.read()
+
+        # Read the destination HTML file
+        with open('args.indexTemplate', 'r', encoding='utf-8') as dest_file:
+            dest_html = dest_file.read()
+
+        # Parse both HTML files
+        source_soup = BeautifulSoup(source_html, 'html.parser')
+        dest_soup = BeautifulSoup(dest_html, 'html.parser')
+
+        # Find the <ul> element in both the source and destination HTML
+        source_ul = source_soup.find('ul')
+        dest_ul = dest_soup.find('ul')
+
+        # Copy the contents of the source <ul> to the destination <ul>
+        dest_ul.extend(source_ul.contents)
+
+        # Save the updated destination HTML
+        with open('public/index.html', 'w', encoding='utf-8') as dest_updated_file:
+            dest_updated_file.write(str(dest_soup))
+
+
+        # Read index HTML
+        with open('public/index.html', 'r', encoding='utf-8') as file:
+            html_content = file.read()
+        print("Index.html upgraded")
+
+
 def updateIndex(args:argparse.Namespace) -> None:
    
    # Check if index.htm exists
@@ -63,10 +100,15 @@ if __name__ == '__main__':
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-re', '--removeEntry', action='store_true', required=False, default=False,
                        help="Indicate to remove the given entry")
+    parser.add_argument('-u', '--upgrade', action='store_true', required=False, default=False,
+                        help="Upgrade the index.html")
    parser.add_argument('documentName', help = 'Document name to add to the list')
    parser.add_argument('documentLink', help = 'Document location to add to the list')
    parser.add_argument('indexTemplate', help = 'Index template document')
    args = parser.parse_args()

+    if args.upgrade:
+        upgradeIndex(args)
+    else:
        updateIndex(args)

--- a/upgrade_pages.sh
+++ b/upgrade_pages.sh
+#!/bin/bash
+
+GITLAB_HOST="http://git.onem2m.org"
+SPECIFICATIONS_GROUP="2501"
+NS_GITLAB_HOST=$(echo $GITLAB_HOST | cut -d'/' -f3)
+
+echo "\n------ Getting all projects IDs --------"
+# Get a list of projects in the group using GitLab API
+PROJECT_IDS=$(curl --header "PRIVATE-TOKEN: $1" "$GITLAB_HOST/api/v4/groups/$SPECIFICATIONS_GROUP/projects" | grep -oP '"id":(?!$SPECIFICATIONS_GROUP)\d+' | cut -d':' -f2)
+echo $PROJECT_IDS
+
+PROJECT_IDS = "41" # Use TS-0019 for debugging
+# Trigger pipeline for each project. Triggered pipeline pages job will handle the upgrade of the index.html
+for PROJECT_ID in $PROJECT_IDS; do
+	echo "------ Preparing specification project --------"
+    curl -X POST --fail -F token=$ACCESS_TOKEN -F ref=REF_NAME -F "https://git.onem2m.org/api/v4/projects/$PROJECT_ID/trigger/pipeline"
+done
+
+exit 0
\ No newline at end of file